mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge branch 'main' of https://github.com/BerriAI/litellm into litellm_fix_model_cost_reload_race
# Conflicts: # tests/test_litellm/test_utils.py
This commit is contained in:
commit
4de7bf4ee0
20 changed files with 766 additions and 404 deletions
|
|
@ -35,11 +35,6 @@ from litellm.types.utils import (
|
|||
StandardLoggingGuardrailInformation,
|
||||
)
|
||||
|
||||
try:
|
||||
from fastapi.exceptions import HTTPException
|
||||
except ImportError:
|
||||
HTTPException = None
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
|
||||
|
|
@ -107,9 +102,9 @@ def is_guardrail_intervention(e: Exception) -> bool:
|
|||
),
|
||||
):
|
||||
return True
|
||||
if HTTPException is not None and isinstance(e, HTTPException) and e.status_code in _GUARDRAIL_BLOCK_STATUS_CODES:
|
||||
return True
|
||||
return False
|
||||
from litellm.proxy.guardrails.exception_utils import is_fastapi_http_exception
|
||||
|
||||
return is_fastapi_http_exception(e, _GUARDRAIL_BLOCK_STATUS_CODES)
|
||||
|
||||
|
||||
def _strict_guardrail_modes_enabled() -> bool:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
|
|||
from litellm.litellm_core_utils.cloud_storage_security import (
|
||||
sanitize_cloud_object_component,
|
||||
)
|
||||
from litellm.proxy._types import CommonProxyErrors
|
||||
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
|
||||
from litellm.types.integrations.gcs_bucket import *
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
|
@ -27,6 +26,7 @@ else:
|
|||
|
||||
class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
|
||||
def __init__(self, bucket_name: str | None = None) -> None:
|
||||
from litellm.proxy._types import CommonProxyErrors
|
||||
from litellm.proxy.proxy_server import premium_user
|
||||
|
||||
self.batch_size = int(os.getenv("GCS_BATCH_SIZE", GCS_DEFAULT_BATCH_SIZE))
|
||||
|
|
@ -52,6 +52,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
|
|||
|
||||
#### ASYNC ####
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
from litellm.proxy._types import CommonProxyErrors
|
||||
from litellm.proxy.proxy_server import premium_user
|
||||
|
||||
if premium_user is not True:
|
||||
|
|
|
|||
|
|
@ -519,7 +519,6 @@ def _get_token_base_cost(
|
|||
current_time: datetime | None = None,
|
||||
*,
|
||||
threshold_is_inclusive: bool = False,
|
||||
missing_cache_read_uses_input: bool = False,
|
||||
) -> tuple[float, float, float, float, float]:
|
||||
"""
|
||||
Return prompt cost, completion cost, and cache costs for a given model and usage.
|
||||
|
|
@ -530,13 +529,11 @@ def _get_token_base_cost(
|
|||
`threshold_is_inclusive` switches that comparison to >=, for providers such as xAI
|
||||
that bill the higher tier once the prompt reaches the threshold.
|
||||
|
||||
`missing_cache_read_uses_input` resolves an absent cache-read rate to the resolved
|
||||
input rate instead of 0.0; an explicit 0.0 rate stays a real price either way.
|
||||
|
||||
An absent cache-creation rate always resolves to the resolved input rate, the way the
|
||||
tiered table and custom deployment pricing already do, since a provider that publishes
|
||||
no write price bills cache writes as ordinary input. An absent 1h write rate resolves
|
||||
to the cache-creation rate, off-peak included. An explicit 0.0 stays a real price for both.
|
||||
An absent cache-creation or cache-read rate always resolves to the resolved input
|
||||
rate, the way the tiered table and custom deployment pricing already do, since a
|
||||
provider that publishes no cache price bills cached tokens as ordinary input. An
|
||||
absent 1h write rate resolves to the cache-creation rate, off-peak included. An
|
||||
explicit 0.0 stays a real price for all of them.
|
||||
|
||||
Returns:
|
||||
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
|
||||
|
|
@ -663,8 +660,7 @@ def _get_token_base_cost(
|
|||
"input_cost_per_token",
|
||||
prompt_base_cost,
|
||||
)
|
||||
if cache_read_cost is None:
|
||||
cache_read_cost = input_rate_for_missing_cache_rates if missing_cache_read_uses_input else 0.0
|
||||
resolved_cache_read_cost: Final = input_rate_for_missing_cache_rates if cache_read_cost is None else cache_read_cost
|
||||
resolved_cache_creation_cost: Final = (
|
||||
input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
|
||||
)
|
||||
|
|
@ -677,7 +673,7 @@ def _get_token_base_cost(
|
|||
completion_base_cost,
|
||||
resolved_cache_creation_cost,
|
||||
cache_creation_cost_above_1hr,
|
||||
cache_read_cost,
|
||||
resolved_cache_read_cost,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -1588,7 +1584,6 @@ def calculate_prompt_caching_savings(
|
|||
service_tier=service_tier,
|
||||
current_time=billed_at,
|
||||
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
|
||||
missing_cache_read_uses_input=True,
|
||||
)
|
||||
write_rate: Final = cache_creation_cost or prompt_base_cost
|
||||
write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from typing_extensions import ParamSpec, TypeVar
|
|||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm._lazy_imports import _get_default_encoding
|
||||
from litellm.constants import (
|
||||
DEFAULT_IMAGE_HEIGHT,
|
||||
DEFAULT_IMAGE_TOKEN_COUNT,
|
||||
|
|
@ -29,7 +30,6 @@ from litellm.constants import (
|
|||
TOKEN_COUNTER_MAX_EXACT_CHARS,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.default_encoding import encoding as default_encoding
|
||||
from litellm.litellm_core_utils.url_utils import safe_get
|
||||
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
|
||||
from litellm.types.llms.anthropic import (
|
||||
|
|
@ -638,7 +638,7 @@ def _get_exact_count_function(
|
|||
else:
|
||||
|
||||
def encode_length(text: str) -> int:
|
||||
return len(default_encoding.encode(text, disallowed_special=()))
|
||||
return len(_get_default_encoding().encode(text, disallowed_special=()))
|
||||
|
||||
return _get_tiktoken_count_function(encode_length)
|
||||
|
||||
|
|
|
|||
9
litellm/proxy/guardrails/exception_utils.py
Normal file
9
litellm/proxy/guardrails/exception_utils.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
from collections.abc import Collection
|
||||
|
||||
|
||||
def is_fastapi_http_exception(e: Exception, block_status_codes: Collection[int]) -> bool:
|
||||
try:
|
||||
from fastapi.exceptions import HTTPException
|
||||
except ImportError:
|
||||
return False
|
||||
return isinstance(e, HTTPException) and e.status_code in block_status_codes
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import base64
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from functools import reduce
|
||||
from typing import Any, Final, Optional, TypeVar, Union, cast, get_type_hints, overload
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -8,6 +9,7 @@ from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire pay
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -29,6 +31,11 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
|
||||
def _apply_nested_drop_params(params: dict[str, object], additional_drop_params: list[str] | None) -> dict[str, object]:
|
||||
nested_paths: Final = tuple(path for path in additional_drop_params or () if is_nested_path(path))
|
||||
return reduce(lambda acc, path: delete_nested_value(acc, path), nested_paths, params)
|
||||
|
||||
|
||||
def _output_token_detail(details: object, field: str) -> int | None:
|
||||
value: Final = getattr(details, field, None)
|
||||
return value if isinstance(value, int) else None
|
||||
|
|
@ -265,20 +272,24 @@ class ResponsesAPIRequestUtils:
|
|||
special_params: Final[dict[str, object]] = params.pop("kwargs", {})
|
||||
|
||||
additional_drop_params: Final[list[str] | None] = params.pop("additional_drop_params", None)
|
||||
non_default_params: Final = PreProcessNonDefaultParams.base_pre_process_non_default_params(
|
||||
passed_params=params,
|
||||
special_params=special_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
additional_drop_params=additional_drop_params,
|
||||
default_param_values={k: None for k in valid_keys},
|
||||
additional_endpoint_specific_params=["input"],
|
||||
non_default_params: Final = _apply_nested_drop_params(
|
||||
PreProcessNonDefaultParams.base_pre_process_non_default_params(
|
||||
passed_params=params,
|
||||
special_params=special_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
additional_drop_params=additional_drop_params,
|
||||
default_param_values={k: None for k in valid_keys},
|
||||
additional_endpoint_specific_params=["input"],
|
||||
),
|
||||
additional_drop_params,
|
||||
)
|
||||
|
||||
# decode previous_response_id if it's a litellm encoded id
|
||||
if "previous_response_id" in non_default_params:
|
||||
previous_response_id: Final = non_default_params.get("previous_response_id")
|
||||
if isinstance(previous_response_id, str):
|
||||
decoded_previous_response_id: Final = (
|
||||
ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(
|
||||
non_default_params["previous_response_id"]
|
||||
previous_response_id
|
||||
)
|
||||
)
|
||||
non_default_params["previous_response_id"] = decoded_previous_response_id
|
||||
|
|
@ -286,7 +297,8 @@ class ResponsesAPIRequestUtils:
|
|||
if "metadata" in non_default_params:
|
||||
from litellm.utils import add_openai_metadata
|
||||
|
||||
converted_metadata: Final = add_openai_metadata(non_default_params["metadata"])
|
||||
raw_metadata: Final = non_default_params["metadata"]
|
||||
converted_metadata: Final = add_openai_metadata(raw_metadata if _is_object_dict(raw_metadata) else None)
|
||||
if converted_metadata is not None:
|
||||
non_default_params["metadata"] = converted_metadata
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -155,7 +155,7 @@ from litellm.router_utils.batch_utils import (
|
|||
replace_model_in_jsonl,
|
||||
should_replace_model_in_jsonl,
|
||||
)
|
||||
from litellm.router_utils.client_initalization_utils import InitalizeCachedClient
|
||||
from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit
|
||||
from litellm.router_utils.clientside_credential_handler import (
|
||||
get_dynamic_litellm_params,
|
||||
is_clientside_credential,
|
||||
|
|
@ -3642,24 +3642,22 @@ class Router:
|
|||
input_kwargs.pop("silent_model", None)
|
||||
input_kwargs.pop("include_fallback_errors", None)
|
||||
|
||||
_response: Final = litellm.acompletion(**input_kwargs)
|
||||
|
||||
logging_obj: Final[LiteLLMLogging | None] = kwargs.get("litellm_logging_obj", None)
|
||||
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
max_parallel_requests_limit: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
async with contextlib.AsyncExitStack() as deployment_slot:
|
||||
if isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
await deployment_slot.enter_async_context(rpm_semaphore)
|
||||
if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit):
|
||||
deployment_slot.enter_context(max_parallel_requests_limit)
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment,
|
||||
logging_obj=logging_obj,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
response = await _response
|
||||
response = await litellm.acompletion(**input_kwargs)
|
||||
|
||||
## CHECK CONTENT FILTER ERROR ##
|
||||
if isinstance(response, ModelResponse):
|
||||
|
|
@ -4586,38 +4584,16 @@ class Router:
|
|||
)
|
||||
|
||||
self.total_calls[model_name] += 1
|
||||
response = litellm.aimage_generation(
|
||||
**{
|
||||
**data,
|
||||
"prompt": prompt,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
### CONCURRENCY-SAFE RPM CHECKS ###
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
"""
|
||||
- Check rpm limits before making the call
|
||||
- If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe)
|
||||
"""
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await litellm.aimage_generation(
|
||||
**{
|
||||
**data,
|
||||
"prompt": prompt,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
response = await response
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.aimage_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
|
@ -4691,38 +4667,16 @@ class Router:
|
|||
)
|
||||
|
||||
self.total_calls[model_name] += 1
|
||||
response = litellm.atranscription(
|
||||
**{
|
||||
**data,
|
||||
"file": file,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
### CONCURRENCY-SAFE RPM CHECKS ###
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
"""
|
||||
- Check rpm limits before making the call
|
||||
- If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe)
|
||||
"""
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await litellm.atranscription(
|
||||
**{
|
||||
**data,
|
||||
"file": file,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
response = await response
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.atranscription(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
|
@ -4806,38 +4760,16 @@ class Router:
|
|||
)
|
||||
|
||||
self.total_calls[model_name] += 1
|
||||
response = litellm.aspeech(
|
||||
**{
|
||||
**data,
|
||||
"input": input,
|
||||
"voice": data.get("voice") if voice is None else voice,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
### CONCURRENCY-SAFE RPM CHECKS ###
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
"""
|
||||
- Check rpm limits before making the call
|
||||
- If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe)
|
||||
"""
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await litellm.aspeech(
|
||||
**{
|
||||
**data,
|
||||
"input": input,
|
||||
"voice": data.get("voice") if voice is None else voice,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
response = await response
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.aspeech(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
|
@ -5002,37 +4934,16 @@ class Router:
|
|||
)
|
||||
self.total_calls[model_name] += 1
|
||||
|
||||
response = litellm.atext_completion(
|
||||
**{
|
||||
**data,
|
||||
"prompt": prompt,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
"""
|
||||
- Check rpm limits before making the call
|
||||
- If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe)
|
||||
"""
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await litellm.atext_completion(
|
||||
**{
|
||||
**data,
|
||||
"prompt": prompt,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
response = await response
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.atext_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
|
@ -5093,37 +5004,16 @@ class Router:
|
|||
)
|
||||
self.total_calls[model_name] += 1
|
||||
|
||||
response = litellm.aadapter_completion(
|
||||
**{
|
||||
**data,
|
||||
"adapter_id": adapter_id,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
"""
|
||||
- Check rpm limits before making the call
|
||||
- If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe)
|
||||
"""
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await litellm.aadapter_completion(
|
||||
**{
|
||||
**data,
|
||||
"adapter_id": adapter_id,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
response = await response
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.aadapter_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
|
@ -5353,29 +5243,8 @@ class Router:
|
|||
if custom_llm_provider is not None:
|
||||
response_kwargs["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
response = original_generic_function(**response_kwargs)
|
||||
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
"""
|
||||
- Check rpm limits before making the call
|
||||
- If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe)
|
||||
"""
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await original_generic_function(**response_kwargs)
|
||||
|
||||
if self._should_raise_anthropic_refusal_error(
|
||||
model=model,
|
||||
|
|
@ -5983,38 +5852,16 @@ class Router:
|
|||
)
|
||||
|
||||
self.total_calls[model_name] += 1
|
||||
response = litellm.aembedding(
|
||||
**{
|
||||
**data,
|
||||
"input": input,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
### CONCURRENCY-SAFE RPM CHECKS ###
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
"""
|
||||
- Check rpm limits before making the call
|
||||
- If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe)
|
||||
"""
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await litellm.aembedding(
|
||||
**{
|
||||
**data,
|
||||
"input": input,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
response = await response
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.aembedding(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
|
@ -6123,37 +5970,18 @@ class Router:
|
|||
"gcs_bucket_name" in data
|
||||
): # TODO: Remove this once we have a better way to handle GCS bucket name: Problem is that we need to pass the gcs_bucket_name to the router for the create_file call but it doesn't show up there
|
||||
kwargs_copy.setdefault("litellm_metadata", {})["gcs_bucket_name"] = data["gcs_bucket_name"]
|
||||
response = litellm.acreate_file(
|
||||
**{
|
||||
**data,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs_copy,
|
||||
}
|
||||
)
|
||||
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs_copy,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
"""
|
||||
- Check rpm limits before making the call
|
||||
- If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe)
|
||||
"""
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
async with self._deployment_slot(
|
||||
deployment=deployment, kwargs=kwargs_copy, parent_otel_span=parent_otel_span
|
||||
):
|
||||
response = await litellm.acreate_file(
|
||||
**{
|
||||
**data,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs_copy,
|
||||
}
|
||||
)
|
||||
response = await response
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.acreate_file(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
|
@ -6243,33 +6071,16 @@ class Router:
|
|||
)
|
||||
custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider
|
||||
|
||||
response = avector_store_create_sdk(
|
||||
**{
|
||||
**data,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await avector_store_create_sdk(
|
||||
**{
|
||||
**data,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
response = await response
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.avector_store_create(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
|
@ -6355,37 +6166,16 @@ class Router:
|
|||
)
|
||||
custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider
|
||||
|
||||
response = litellm.acreate_batch(
|
||||
**{
|
||||
**data,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
"""
|
||||
- Check rpm limits before making the call
|
||||
- If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe)
|
||||
"""
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await litellm.acreate_batch(
|
||||
**{
|
||||
**data,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
response = await response
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.acreate_batch(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
|
@ -6576,37 +6366,16 @@ class Router:
|
|||
)
|
||||
custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider
|
||||
|
||||
response = litellm.acancel_batch(
|
||||
**{
|
||||
**data,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
rpm_semaphore: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore):
|
||||
async with rpm_semaphore:
|
||||
"""
|
||||
- Check rpm limits before making the call
|
||||
- If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe)
|
||||
"""
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
)
|
||||
response = await response
|
||||
else:
|
||||
await self.async_routing_strategy_pre_call_checks(
|
||||
deployment=deployment, parent_otel_span=parent_otel_span
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await litellm.acancel_batch(
|
||||
**{
|
||||
**data,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"caching": self.cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
response = await response
|
||||
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.acancel_batch(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
|
@ -8741,6 +8510,23 @@ class Router:
|
|||
)
|
||||
raise e
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _deployment_slot(
|
||||
self, deployment: dict, kwargs: Mapping[str, object], parent_otel_span: Span | None
|
||||
) -> AsyncGenerator[None, None]:
|
||||
"""Holds the deployment's max_parallel_requests slot, if it has one, around the provider call. Routing
|
||||
strategy pre-call checks run inside the slot so their rpm accounting stays concurrency-safe."""
|
||||
max_parallel_requests_limit: Final = self._get_client(
|
||||
deployment=deployment,
|
||||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
async with contextlib.AsyncExitStack() as slot:
|
||||
if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit):
|
||||
slot.enter_context(max_parallel_requests_limit)
|
||||
await self.async_routing_strategy_pre_call_checks(deployment=deployment, parent_otel_span=parent_otel_span)
|
||||
yield
|
||||
|
||||
async def async_callback_filter_deployments(
|
||||
self,
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import asyncio
|
||||
from types import TracebackType
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from litellm.exceptions import RateLimitError, RateLimitErrorCategory, RateLimitType
|
||||
from litellm.types.router import RouterErrors
|
||||
from litellm.utils import calculate_max_parallel_requests
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -11,6 +13,43 @@ else:
|
|||
LitellmRouter = Any
|
||||
|
||||
|
||||
class MaxParallelRequestsLimit:
|
||||
"""A deployment's max_parallel_requests slots. A caller arriving while every slot is in use gets a 429 instead
|
||||
of waiting for one to free up."""
|
||||
|
||||
def __init__(self, max_parallel_requests: int, model_id: str, model_group: str) -> None:
|
||||
self.max_parallel_requests: Final = max_parallel_requests
|
||||
self.model_id: Final = model_id
|
||||
self.model_group: Final = model_group
|
||||
self.in_flight = 0
|
||||
|
||||
def __enter__(self) -> None:
|
||||
self.acquire()
|
||||
|
||||
def __exit__(
|
||||
self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None
|
||||
) -> None:
|
||||
self.release()
|
||||
|
||||
def acquire(self) -> None:
|
||||
if self.in_flight >= self.max_parallel_requests:
|
||||
raise RateLimitError(
|
||||
message=(
|
||||
f"{RouterErrors.max_parallel_requests_exceeded.value} Deployment model_group={self.model_group}, "
|
||||
f"id={self.model_id} already has max_parallel_requests={self.max_parallel_requests} requests in "
|
||||
"flight. Raise max_parallel_requests (or the rpm/tpm it is derived from) for this deployment"
|
||||
),
|
||||
llm_provider="",
|
||||
model=self.model_group,
|
||||
category=RateLimitErrorCategory.LITELLM_RATE_LIMIT,
|
||||
rate_limit_type=RateLimitType.CONCURRENT_REQUESTS,
|
||||
)
|
||||
self.in_flight += 1
|
||||
|
||||
def release(self) -> None:
|
||||
self.in_flight -= 1
|
||||
|
||||
|
||||
class InitalizeCachedClient:
|
||||
@staticmethod
|
||||
def set_max_parallel_requests_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||
|
|
@ -26,10 +65,14 @@ class InitalizeCachedClient:
|
|||
default_max_parallel_requests=litellm_router_instance.default_max_parallel_requests,
|
||||
)
|
||||
if calculated_max_parallel_requests:
|
||||
semaphore: Final = asyncio.Semaphore(calculated_max_parallel_requests)
|
||||
limit: Final = MaxParallelRequestsLimit(
|
||||
max_parallel_requests=calculated_max_parallel_requests,
|
||||
model_id=model_id,
|
||||
model_group=model.get("model_name", ""),
|
||||
)
|
||||
cache_key: Final = f"{model_id}_max_parallel_requests_client"
|
||||
litellm_router_instance.cache.set_cache(
|
||||
key=cache_key,
|
||||
value=semaphore,
|
||||
value=limit,
|
||||
local_only=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -653,6 +653,7 @@ class RouterErrors(enum.Enum):
|
|||
"""
|
||||
|
||||
user_defined_ratelimit_error = "Deployment over user-defined ratelimit."
|
||||
max_parallel_requests_exceeded = "Deployment has all max_parallel_requests slots in use."
|
||||
no_deployments_available = "No deployments available for selected model"
|
||||
all_deployments_in_cooldown = "All deployments for selected model are in cooldown"
|
||||
no_deployments_with_tag_routing = "Not allowed to access model due to tags configuration"
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import pytest
|
|||
from typing import Optional
|
||||
|
||||
import litellm
|
||||
from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
|
||||
from litellm.utils import calculate_max_parallel_requests
|
||||
|
||||
"""
|
||||
|
|
@ -93,26 +94,26 @@ def test_setting_mpr_limits_per_model(
|
|||
default_max_parallel_requests=default_max_parallel_requests,
|
||||
)
|
||||
|
||||
mpr_client: Optional[asyncio.Semaphore] = router._get_client(
|
||||
mpr_client: Optional[MaxParallelRequestsLimit] = router._get_client(
|
||||
deployment=deployment,
|
||||
kwargs={},
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
|
||||
if max_parallel_requests is not None:
|
||||
assert max_parallel_requests == mpr_client._value
|
||||
assert max_parallel_requests == mpr_client.max_parallel_requests
|
||||
elif rpm is not None:
|
||||
assert rpm == mpr_client._value
|
||||
assert rpm == mpr_client.max_parallel_requests
|
||||
elif tpm is not None:
|
||||
calculated_rpm = int(tpm / 1000 * 6)
|
||||
if calculated_rpm == 0:
|
||||
calculated_rpm = 1
|
||||
print(
|
||||
f"test calculated_rpm: {calculated_rpm}, calculated_max_parallel_requests={mpr_client._value}"
|
||||
f"test calculated_rpm: {calculated_rpm}, calculated_max_parallel_requests={mpr_client.max_parallel_requests}"
|
||||
)
|
||||
assert calculated_rpm == mpr_client._value
|
||||
assert calculated_rpm == mpr_client.max_parallel_requests
|
||||
elif default_max_parallel_requests is not None:
|
||||
assert mpr_client._value == default_max_parallel_requests
|
||||
assert mpr_client.max_parallel_requests == default_max_parallel_requests
|
||||
else:
|
||||
assert mpr_client is None
|
||||
|
||||
|
|
|
|||
|
|
@ -131,6 +131,13 @@ class TestGCSBucketBase:
|
|||
|
||||
|
||||
class TestGCSBucketLoggerBucketName:
|
||||
@pytest.mark.asyncio
|
||||
async def test_constructor_rejects_non_premium_user(self, monkeypatch):
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False)
|
||||
|
||||
with pytest.raises(ValueError, match="GCS Bucket logging is a premium feature"):
|
||||
GCSBucketLogger(bucket_name="config-bucket")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_bucket_name_it_is_constructed_with_survives(self, monkeypatch):
|
||||
"""Reading config.yaml out of a GCS bucket asks for that bucket, not the logging one (LIT-6982)."""
|
||||
|
|
@ -145,3 +152,11 @@ class TestGCSBucketLoggerBucketName:
|
|||
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True)
|
||||
|
||||
assert GCSBucketLogger().BUCKET_NAME == "logging-bucket"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_logging_rejects_non_premium_user(self, monkeypatch):
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False)
|
||||
logger = object.__new__(GCSBucketLogger)
|
||||
|
||||
with pytest.raises(ValueError, match="GCS Bucket logging is a premium feature"):
|
||||
await logger.async_log_success_event({}, None, None, None)
|
||||
|
|
|
|||
|
|
@ -1754,6 +1754,20 @@ class TestCustomGuardrailSpendLogMatchRedaction:
|
|||
class TestGuardrailInterventionClassification:
|
||||
"""A routing decision is a deliberate guardrail intervention, not a failure."""
|
||||
|
||||
def test_http_exception_classification_returns_false_without_fastapi(self, monkeypatch):
|
||||
import builtins
|
||||
|
||||
real_import = builtins.__import__
|
||||
|
||||
def import_without_fastapi(name, *args, **kwargs):
|
||||
if name == "fastapi.exceptions":
|
||||
raise ImportError("fastapi is unavailable")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", import_without_fastapi)
|
||||
|
||||
assert CustomGuardrail._is_guardrail_intervention(Exception("not an intervention")) is False
|
||||
|
||||
def test_sensitive_data_route_exception_is_intervention(self):
|
||||
from litellm.exceptions import SensitiveDataRouteException
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
@pytest.mark.parametrize("prompt_tokens", [100, 200000, 200001])
|
||||
@pytest.mark.parametrize("read_rate", [None, 0.0, 0.25e-6])
|
||||
@pytest.mark.parametrize("service_tier", [None, "priority"])
|
||||
def test_missing_cache_read_policy_preserves_billing(prompt_tokens, read_rate, service_tier):
|
||||
def test_missing_cache_read_rate_resolves_to_input_rate(prompt_tokens, read_rate, service_tier):
|
||||
info = {
|
||||
"input_cost_per_token": 3e-6,
|
||||
"input_cost_per_token_priority": 4e-6,
|
||||
|
|
@ -59,16 +59,43 @@ def test_missing_cache_read_policy_preserves_billing(prompt_tokens, read_rate, s
|
|||
}
|
||||
usage = Usage(prompt_tokens=prompt_tokens, prompt_tokens_details={"cached_tokens": 100})
|
||||
billed = _get_token_base_cost(info, usage, service_tier=service_tier)
|
||||
savings = _get_token_base_cost(info, usage, service_tier=service_tier, missing_cache_read_uses_input=True)
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
"policy-fixture", usage, "openai", service_tier=service_tier, model_info=info
|
||||
)
|
||||
assert billed[4] == pytest.approx(read_rate or 0.0)
|
||||
assert savings[:4] == billed[:4]
|
||||
assert savings[4] == pytest.approx(billed[0] if read_rate is None else read_rate)
|
||||
assert billed[4] == pytest.approx(read_rate if read_rate is not None else billed[0])
|
||||
assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4])
|
||||
|
||||
|
||||
def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None:
|
||||
model_info: ModelInfo = {
|
||||
"key": "bare-model",
|
||||
"max_tokens": None,
|
||||
"max_input_tokens": None,
|
||||
"max_output_tokens": None,
|
||||
"input_cost_per_token": 2.4e-7,
|
||||
"output_cost_per_token": 9.7e-7,
|
||||
"litellm_provider": "bedrock",
|
||||
"mode": "chat",
|
||||
"supported_openai_params": None,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=12928,
|
||||
completion_tokens=380,
|
||||
total_tokens=13308,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=12288),
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model="bare-model",
|
||||
usage=usage,
|
||||
custom_llm_provider="bedrock",
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(12928 * 2.4e-7)
|
||||
assert completion_cost == pytest.approx(380 * 9.7e-7)
|
||||
|
||||
|
||||
def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None:
|
||||
model_info: ModelInfo = {
|
||||
"key": "gemini-embedding-2",
|
||||
|
|
@ -180,11 +207,7 @@ def test_missing_cache_read_uses_off_peak_input_rate():
|
|||
}
|
||||
when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc)
|
||||
billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when)
|
||||
savings = _get_token_base_cost(
|
||||
info, Usage(prompt_tokens=100), current_time=when, missing_cache_read_uses_input=True
|
||||
)
|
||||
assert billed[4] == 0.0
|
||||
assert savings[0] == savings[4] == 5e-6
|
||||
assert billed[0] == billed[4] == 5e-6
|
||||
|
||||
|
||||
def test_reasoning_tokens_no_price_set(_local_model_cost_map):
|
||||
|
|
|
|||
|
|
@ -98,6 +98,13 @@ def test_token_counter_short_text_matches_tiktoken(text):
|
|||
assert token_counter_new(model="us.anthropic.claude-sonnet-4-6", text=text) == expected
|
||||
|
||||
|
||||
def test_token_counter_default_encoding_matches_cl100k():
|
||||
encoding: Final = tiktoken.get_encoding("cl100k_base")
|
||||
expected: Final = len(encoding.encode("hello world", disallowed_special=()))
|
||||
|
||||
assert token_counter_new(model=None, text="hello world") == expected
|
||||
|
||||
|
||||
def test_token_counter_text_over_chunk_boundary_stays_close_to_tiktoken():
|
||||
text = ("The quick brown fox jumps over the lazy dog. " * 30)[:1025]
|
||||
encoding = tiktoken.get_encoding("cl100k_base")
|
||||
|
|
|
|||
|
|
@ -975,9 +975,9 @@ class TestEstimateCostCacheAndReasoningTokens:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_model_without_cache_or_reasoning_prices_estimates_what_the_proxy_bills(self, monkeypatch):
|
||||
"""The cost calculator bills cache reads of a cost-map model without cache prices at zero,
|
||||
its cache writes at the input rate, and its reasoning tokens at the output rate. The estimate
|
||||
reports those effective rates."""
|
||||
"""The cost calculator bills cache reads and writes of a cost-map model without cache prices
|
||||
at the input rate, and its reasoning tokens at the output rate. The estimate reports those
|
||||
effective rates."""
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
A_MAPPED_MODEL,
|
||||
|
|
@ -986,14 +986,12 @@ class TestEstimateCostCacheAndReasoningTokens:
|
|||
|
||||
response = await _estimate_with_cache_and_reasoning(None, model=A_MAPPED_MODEL)
|
||||
|
||||
assert response.cache_read_cost_per_request == 0.0
|
||||
assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 5e-6)
|
||||
assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 5e-6)
|
||||
assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 6e-6)
|
||||
assert response.input_cost_per_request == pytest.approx((TEXT_INPUT_TOKENS + CACHE_CREATION_TOKENS) * 5e-6)
|
||||
assert response.cost_per_request == pytest.approx(
|
||||
(TEXT_INPUT_TOKENS + CACHE_CREATION_TOKENS) * 5e-6 + OUTPUT_TOKENS * 6e-6
|
||||
)
|
||||
assert response.cache_read_input_token_cost == 0.0
|
||||
assert response.input_cost_per_request == pytest.approx(INPUT_TOKENS * 5e-6)
|
||||
assert response.cost_per_request == pytest.approx(INPUT_TOKENS * 5e-6 + OUTPUT_TOKENS * 6e-6)
|
||||
assert response.cache_read_input_token_cost == pytest.approx(5e-6)
|
||||
assert response.cache_creation_input_token_cost == pytest.approx(5e-6)
|
||||
assert response.output_cost_per_reasoning_token == pytest.approx(6e-6)
|
||||
|
||||
|
|
|
|||
|
|
@ -119,6 +119,63 @@ class TestResponsesAPIRequestUtils:
|
|||
assert result["max_output_tokens"] == 100
|
||||
assert result["prompt"] == {"id": "pmpt_456"}
|
||||
|
||||
def test_get_requested_response_api_optional_param_drops_nested_path(self):
|
||||
"""Nested additional_drop_params paths like reasoning.summary must be honored"""
|
||||
params = {
|
||||
"temperature": 0.1,
|
||||
"reasoning": {"effort": "high", "summary": "auto"},
|
||||
"additional_drop_params": ["reasoning.summary"],
|
||||
}
|
||||
|
||||
result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params)
|
||||
|
||||
assert result["reasoning"] == {"effort": "high"}
|
||||
assert result["temperature"] == 0.1
|
||||
|
||||
def test_get_requested_response_api_optional_param_drops_array_path(self):
|
||||
"""Array wildcard paths like tools[*].input_examples must be honored"""
|
||||
params = {
|
||||
"tools": [{"type": "function", "name": "t", "input_examples": ["x"]}],
|
||||
"additional_drop_params": ["tools[*].input_examples"],
|
||||
}
|
||||
|
||||
result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params)
|
||||
|
||||
assert result["tools"] == [{"type": "function", "name": "t"}]
|
||||
|
||||
def test_get_requested_response_api_optional_param_drops_top_level(self):
|
||||
"""Top-level additional_drop_params keys must still be honored"""
|
||||
params = {
|
||||
"reasoning": {"effort": "high", "summary": "auto"},
|
||||
"additional_drop_params": ["reasoning"],
|
||||
}
|
||||
|
||||
result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params)
|
||||
|
||||
assert "reasoning" not in result
|
||||
|
||||
def test_get_requested_response_api_optional_param_non_matching_nested_path(self):
|
||||
"""A nested path that does not match anything leaves params untouched"""
|
||||
params = {
|
||||
"reasoning": {"effort": "high", "summary": "auto"},
|
||||
"additional_drop_params": ["reasoning.nope"],
|
||||
}
|
||||
|
||||
result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params)
|
||||
|
||||
assert result["reasoning"] == {"effort": "high", "summary": "auto"}
|
||||
|
||||
def test_get_requested_response_api_optional_param_none_drop_params(self):
|
||||
"""additional_drop_params=None is a no-op"""
|
||||
params = {
|
||||
"reasoning": {"effort": "high", "summary": "auto"},
|
||||
"additional_drop_params": None,
|
||||
}
|
||||
|
||||
result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params)
|
||||
|
||||
assert result["reasoning"] == {"effort": "high", "summary": "auto"}
|
||||
|
||||
def test_decode_previous_response_id_to_original_previous_response_id(self):
|
||||
"""Test decoding a LiteLLM encoded previous_response_id to the original previous_response_id"""
|
||||
# Setup
|
||||
|
|
|
|||
|
|
@ -0,0 +1,125 @@
|
|||
import asyncio
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
|
||||
|
||||
|
||||
def _limit(max_parallel_requests: int = 1) -> MaxParallelRequestsLimit:
|
||||
return MaxParallelRequestsLimit(
|
||||
max_parallel_requests=max_parallel_requests, model_id="deployment-1", model_group="gpt-5.6"
|
||||
)
|
||||
|
||||
|
||||
async def _hold(limit: MaxParallelRequestsLimit, release: asyncio.Event) -> str:
|
||||
with limit:
|
||||
await release.wait()
|
||||
return "ok"
|
||||
|
||||
|
||||
def _expect_rejection(limit: MaxParallelRequestsLimit) -> litellm.RateLimitError:
|
||||
with pytest.raises(litellm.RateLimitError) as excinfo:
|
||||
limit.acquire()
|
||||
return excinfo.value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_arriving_while_every_slot_is_in_use_gets_429_without_waiting():
|
||||
limit: Final = _limit(max_parallel_requests=2)
|
||||
release: Final = asyncio.Event()
|
||||
holders: Final = [asyncio.create_task(_hold(limit, release)) for _ in range(2)]
|
||||
await asyncio.sleep(0)
|
||||
assert limit.in_flight == 2
|
||||
|
||||
rejection: Final = _expect_rejection(limit)
|
||||
|
||||
assert rejection.status_code == 429
|
||||
assert "deployment-1" in rejection.message
|
||||
assert "gpt-5.6" in rejection.message
|
||||
assert "max_parallel_requests=2" in rejection.message
|
||||
assert limit.in_flight == 2
|
||||
|
||||
release.set()
|
||||
assert await asyncio.wait_for(asyncio.gather(*holders), timeout=2) == ["ok", "ok"]
|
||||
assert limit.in_flight == 0
|
||||
with limit:
|
||||
assert limit.in_flight == 1
|
||||
assert limit.in_flight == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_burst_over_the_cap_admits_exactly_max_parallel_requests_and_rejects_the_rest():
|
||||
limit: Final = _limit(max_parallel_requests=3)
|
||||
release: Final = asyncio.Event()
|
||||
|
||||
async def attempt() -> str:
|
||||
try:
|
||||
return await _hold(limit, release)
|
||||
except litellm.RateLimitError as e:
|
||||
return f"rejected:{e.status_code}"
|
||||
|
||||
callers: Final = [asyncio.create_task(attempt()) for _ in range(10)]
|
||||
await asyncio.sleep(0)
|
||||
assert limit.in_flight == 3
|
||||
release.set()
|
||||
outcomes: Final = await asyncio.wait_for(asyncio.gather(*callers), timeout=2)
|
||||
assert outcomes.count("ok") == 3
|
||||
assert outcomes.count("rejected:429") == 7
|
||||
assert limit.in_flight == 0
|
||||
|
||||
|
||||
def test_slot_is_released_when_the_held_call_raises():
|
||||
limit: Final = _limit()
|
||||
with pytest.raises(RuntimeError):
|
||||
with limit:
|
||||
raise RuntimeError("provider blew up")
|
||||
assert limit.in_flight == 0
|
||||
with limit:
|
||||
assert limit.in_flight == 1
|
||||
|
||||
|
||||
def _router_limit(router: Router, model_name: str) -> MaxParallelRequestsLimit:
|
||||
deployment: Final = router.get_deployment_by_model_group_name(model_group_name=model_name)
|
||||
assert deployment is not None
|
||||
client: Final = router._get_client(
|
||||
deployment=deployment.model_dump(), kwargs={}, client_type="max_parallel_requests"
|
||||
)
|
||||
assert isinstance(client, MaxParallelRequestsLimit)
|
||||
return client
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_params", "expected_cap"),
|
||||
[
|
||||
({"max_parallel_requests": 2, "rpm": 7, "tpm": 100_000}, 2),
|
||||
({"rpm": 7, "tpm": 100_000}, 7),
|
||||
({"tpm": 100_000}, 600),
|
||||
({"tpm": 100}, 1),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_deployment_rejects_past_its_derived_cap(litellm_params: dict[str, int], expected_cap: int):
|
||||
router: Final = Router(
|
||||
model_list=[{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6", **litellm_params}}]
|
||||
)
|
||||
limit: Final = _router_limit(router, "gpt-5.6")
|
||||
assert limit.max_parallel_requests == expected_cap
|
||||
release: Final = asyncio.Event()
|
||||
holders: Final = [asyncio.create_task(_hold(limit, release)) for _ in range(expected_cap)]
|
||||
await asyncio.sleep(0)
|
||||
assert limit.in_flight == expected_cap
|
||||
assert f"max_parallel_requests={expected_cap}" in _expect_rejection(limit).message
|
||||
release.set()
|
||||
assert await asyncio.wait_for(asyncio.gather(*holders), timeout=2) == ["ok"] * expected_cap
|
||||
|
||||
|
||||
def test_router_without_any_concurrency_setting_has_no_limit():
|
||||
router: Final = Router(model_list=[{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6"}}])
|
||||
deployment: Final = router.get_deployment_by_model_group_name(model_group_name="gpt-5.6")
|
||||
assert deployment is not None
|
||||
assert (
|
||||
router._get_client(deployment=deployment.model_dump(), kwargs={}, client_type="max_parallel_requests") is None
|
||||
)
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
"""Simple tests for lazy import functionality."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -38,6 +41,22 @@ from litellm._lazy_imports import (
|
|||
)
|
||||
|
||||
|
||||
def test_import_litellm_does_not_load_fastapi_or_bpe_table():
|
||||
result: Final = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.litellm_core_utils.default_encoding') if m in sys.modules))",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"},
|
||||
)
|
||||
|
||||
assert result.stdout.strip() == ""
|
||||
|
||||
|
||||
def _clear_names_from_globals(names: tuple):
|
||||
"""Clear all names from litellm globals."""
|
||||
# Get the actual globals dict, not a copy
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import functools
|
||||
import gc
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import warnings
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from datetime import datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
|
|
@ -45,6 +47,8 @@ from litellm.router import (
|
|||
_is_retriable_anthropic_status,
|
||||
)
|
||||
from litellm.router_strategy import simple_shuffle
|
||||
from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
|
||||
from litellm.router_utils.cooldown_handlers import _async_get_cooldown_deployments
|
||||
from litellm.types.llms.openai import ChatCompletionRequest
|
||||
from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo, PreRoutingHookResponse, RetryPolicy
|
||||
|
||||
|
|
@ -1517,7 +1521,9 @@ async def test_router_ageneric_api_call_with_fallbacks_helper():
|
|||
},
|
||||
}
|
||||
|
||||
mock_semaphore = asyncio.Semaphore(1)
|
||||
mock_semaphore = MaxParallelRequestsLimit(
|
||||
max_parallel_requests=1, model_id="deployment-1", model_group="gpt-3.5-turbo"
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
router, "_update_kwargs_with_deployment"
|
||||
|
|
@ -15962,7 +15968,7 @@ def _max_parallel_router(max_parallel_requests: int) -> Router:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_router_max_parallel_requests_bounds_in_flight_upstream_calls(
|
||||
async def test_router_max_parallel_requests_admits_the_cap_and_rejects_the_rest_with_429(
|
||||
monkeypatch: pytest.MonkeyPatch, stream: bool
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
|
@ -15988,24 +15994,33 @@ async def test_router_max_parallel_requests_bounds_in_flight_upstream_calls(
|
|||
},
|
||||
)
|
||||
|
||||
async def one_call() -> None:
|
||||
response = await router.acompletion(
|
||||
model="gpt-5.6", messages=[{"role": "user", "content": "hi"}], stream=stream
|
||||
)
|
||||
async def one_call() -> str:
|
||||
try:
|
||||
response = await router.acompletion(
|
||||
model="gpt-5.6", messages=[{"role": "user", "content": "hi"}], stream=stream
|
||||
)
|
||||
except litellm.RateLimitError as e:
|
||||
return f"rejected:{e.status_code}"
|
||||
if stream:
|
||||
async for _ in response:
|
||||
pass
|
||||
return "ok"
|
||||
|
||||
with respx.mock(assert_all_called=True) as respx_mock:
|
||||
respx_mock.post("https://max-parallel.local/v1/chat/completions").mock(side_effect=upstream)
|
||||
await asyncio.wait_for(asyncio.gather(*(one_call() for _ in range(10))), timeout=10)
|
||||
route: Final = respx_mock.post("https://max-parallel.local/v1/chat/completions").mock(side_effect=upstream)
|
||||
outcomes: Final = await asyncio.wait_for(asyncio.gather(*(one_call() for _ in range(10))), timeout=10)
|
||||
|
||||
assert tracker.peak <= 2
|
||||
assert outcomes.count("ok") == 2
|
||||
assert outcomes.count("rejected:429") == 8
|
||||
assert route.call_count == 2
|
||||
assert tracker.peak == 2
|
||||
assert tracker.current == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_max_parallel_requests_slot_released_when_stream_closed_early(monkeypatch: pytest.MonkeyPatch):
|
||||
async def test_router_max_parallel_requests_slot_held_until_stream_closed_then_released(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
tracker: Final = _InFlightTracker()
|
||||
router: Final = _max_parallel_router(max_parallel_requests=1)
|
||||
|
|
@ -16028,16 +16043,232 @@ async def test_router_max_parallel_requests_slot_released_when_stream_closed_ear
|
|||
async for _ in second:
|
||||
pass
|
||||
|
||||
second_task: Final = asyncio.create_task(second_call())
|
||||
await asyncio.sleep(0.05)
|
||||
assert tracker.current == 1
|
||||
with pytest.raises(litellm.RateLimitError) as while_streaming:
|
||||
await second_call()
|
||||
assert while_streaming.value.status_code == 429
|
||||
await first.aclose()
|
||||
await asyncio.wait_for(second_task, timeout=2)
|
||||
await asyncio.wait_for(second_call(), timeout=2)
|
||||
|
||||
assert tracker.peak == 1
|
||||
assert tracker.current == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_max_parallel_requests_overflow_is_429_without_cooldown_or_provider_call(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-5.6",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_key": "sk-fake",
|
||||
"api_base": "https://max-parallel.local/v1",
|
||||
"max_parallel_requests": 1,
|
||||
},
|
||||
"model_info": {"id": "capped-deployment"},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-5.6",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_key": "sk-fake",
|
||||
"api_base": "https://max-parallel-sibling.local/v1",
|
||||
},
|
||||
"model_info": {"id": "sibling-deployment"},
|
||||
},
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
async def upstream(request: httpx.Request) -> httpx.Response:
|
||||
await asyncio.sleep(0.2)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "c",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "x"}, "finish_reason": "stop"}],
|
||||
},
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as respx_mock:
|
||||
route: Final = respx_mock.post("https://max-parallel.local/v1/chat/completions").mock(side_effect=upstream)
|
||||
sibling_route: Final = respx_mock.post("https://max-parallel-sibling.local/v1/chat/completions").mock(
|
||||
side_effect=upstream
|
||||
)
|
||||
results: Final = await asyncio.wait_for(
|
||||
asyncio.gather(
|
||||
*(
|
||||
router.acompletion(model="capped-deployment", messages=[{"role": "user", "content": "hi"}])
|
||||
for _ in range(3)
|
||||
),
|
||||
return_exceptions=True,
|
||||
),
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
rejected: Final = [r for r in results if isinstance(r, BaseException)]
|
||||
assert len(rejected) == 2 and len(results) == 3
|
||||
assert all(isinstance(r, litellm.RateLimitError) and r.status_code == 429 for r in rejected)
|
||||
assert all("capped-deployment" in r.message and "max_parallel_requests=1" in r.message for r in rejected)
|
||||
assert route.call_count == 1
|
||||
assert sibling_route.call_count == 0
|
||||
assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_embedding_path_rejects_past_max_parallel_requests_without_orphan_coroutines(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "embed",
|
||||
"litellm_params": {
|
||||
"model": "openai/text-embedding-3-small",
|
||||
"api_key": "sk-fake",
|
||||
"api_base": "https://max-parallel-embed.local/v1",
|
||||
"max_parallel_requests": 1,
|
||||
},
|
||||
"model_info": {"id": "embed-capped-deployment"},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
async def upstream(request: httpx.Request) -> httpx.Response:
|
||||
await asyncio.sleep(0.2)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
|
||||
"model": "text-embedding-3-small",
|
||||
"usage": {"prompt_tokens": 1, "total_tokens": 1},
|
||||
},
|
||||
)
|
||||
|
||||
with respx.mock() as respx_mock, warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
route: Final = respx_mock.post("https://max-parallel-embed.local/v1/embeddings").mock(side_effect=upstream)
|
||||
results: Final = await asyncio.wait_for(
|
||||
asyncio.gather(
|
||||
*(router.aembedding(model="embed", input=["hi"]) for _ in range(3)),
|
||||
return_exceptions=True,
|
||||
),
|
||||
timeout=10,
|
||||
)
|
||||
gc.collect()
|
||||
|
||||
rejected: Final = [r for r in results if isinstance(r, BaseException)]
|
||||
assert len(rejected) == 2 and len(results) == 3
|
||||
assert all(isinstance(r, litellm.RateLimitError) and r.status_code == 429 for r in rejected)
|
||||
assert all("embed-capped-deployment" in r.message for r in rejected)
|
||||
assert route.call_count == 1
|
||||
assert [str(w.message) for w in caught if "never awaited" in str(w.message)] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_max_parallel_requests_overflow_takes_the_ordinary_429_fallback_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-5.6",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_key": "sk-fake",
|
||||
"api_base": "https://max-parallel-primary.local/v1",
|
||||
"max_parallel_requests": 1,
|
||||
},
|
||||
"model_info": {"id": "capped-primary-deployment"},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-5.6-fallback",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_key": "sk-fake",
|
||||
"api_base": "https://max-parallel-fallback.local/v1",
|
||||
},
|
||||
"model_info": {"id": "fallback-deployment"},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"gpt-5.6": ["gpt-5.6-fallback"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
async def upstream(request: httpx.Request) -> httpx.Response:
|
||||
await asyncio.sleep(0.2)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "c",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "x"}, "finish_reason": "stop"}],
|
||||
},
|
||||
)
|
||||
|
||||
with respx.mock() as respx_mock:
|
||||
primary: Final = respx_mock.post("https://max-parallel-primary.local/v1/chat/completions").mock(
|
||||
side_effect=upstream
|
||||
)
|
||||
fallback: Final = respx_mock.post("https://max-parallel-fallback.local/v1/chat/completions").mock(
|
||||
side_effect=upstream
|
||||
)
|
||||
results: Final = await asyncio.wait_for(
|
||||
asyncio.gather(
|
||||
*(router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) for _ in range(3))
|
||||
),
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
assert len(results) == 3
|
||||
assert primary.call_count == 1
|
||||
assert fallback.call_count == 2
|
||||
assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_deployment_slot_rejects_while_held_and_frees_slot_on_exit():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-5.6",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_key": "sk-fake",
|
||||
"max_parallel_requests": 1,
|
||||
},
|
||||
"model_info": {"id": "slot-deployment"},
|
||||
}
|
||||
]
|
||||
)
|
||||
deployment: Final = router.get_deployment(model_id="slot-deployment")
|
||||
assert deployment is not None
|
||||
kwargs: Final = {"model": "gpt-5.6"}
|
||||
|
||||
async with router._deployment_slot(deployment=deployment.model_dump(), kwargs=kwargs, parent_otel_span=None):
|
||||
with pytest.raises(litellm.RateLimitError) as overflow:
|
||||
async with router._deployment_slot(deployment=deployment.model_dump(), kwargs=kwargs, parent_otel_span=None):
|
||||
pass
|
||||
assert overflow.value.status_code == 429
|
||||
assert "slot-deployment" in overflow.value.message
|
||||
|
||||
async with router._deployment_slot(deployment=deployment.model_dump(), kwargs=kwargs, parent_otel_span=None):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_deployment_drop_params_string_true_is_honored(monkeypatch):
|
||||
from litellm import Router
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ from litellm.utils import (
|
|||
_snapshot_exception_for_hook,
|
||||
async_post_call_failure_deployment_hook,
|
||||
async_post_call_success_deployment_hook,
|
||||
calculate_max_parallel_requests,
|
||||
client,
|
||||
get_non_default_completion_params,
|
||||
get_optional_params_image_gen,
|
||||
|
|
@ -5774,3 +5775,32 @@ def test_register_model_without_a_provider_never_inherits_another_providers_entr
|
|||
assert info["output_cost_per_token"] == 4e-07
|
||||
finally:
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("max_parallel_requests", "rpm", "tpm", "default_max_parallel_requests", "expected"),
|
||||
[
|
||||
(3, 100, 100_000, 7, 3),
|
||||
(None, 100, 100_000, 7, 100),
|
||||
(None, None, 100_000, 7, 600),
|
||||
(None, None, 50, 7, 1),
|
||||
(None, None, None, 7, 7),
|
||||
(None, None, None, None, None),
|
||||
],
|
||||
)
|
||||
def test_calculate_max_parallel_requests_precedence(
|
||||
max_parallel_requests: int | None,
|
||||
rpm: int | None,
|
||||
tpm: int | None,
|
||||
default_max_parallel_requests: int | None,
|
||||
expected: int | None,
|
||||
) -> None:
|
||||
assert (
|
||||
calculate_max_parallel_requests(
|
||||
max_parallel_requests=max_parallel_requests,
|
||||
rpm=rpm,
|
||||
tpm=tpm,
|
||||
default_max_parallel_requests=default_max_parallel_requests,
|
||||
)
|
||||
== expected
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue