mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): keep a deployment's own litellm_params over router default_litellm_params
Router defaults now fill only the keys a deployment leaves unset, so a deployment's own value (vertex_location, api_base, drop_params) beats default_litellm_params. The proxy token counters, drop-params checks, and the Entra cooldown check use the same order
This commit is contained in:
parent
752eb7c5b7
commit
20566c0eed
18 changed files with 416 additions and 95 deletions
|
|
@ -193,6 +193,7 @@ from litellm.responses.streaming_iterator import partial_responses_incomplete_ev
|
|||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
|
||||
from litellm.types.extraction import ExtractionResponse
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.router import RouterRateLimitError
|
||||
|
|
@ -869,14 +870,16 @@ def _router_defaults(llm_router: Router | None) -> Mapping[str, object]:
|
|||
def counted_drop_policies(
|
||||
data: Mapping[str, object], llm_router: Router | None, team_id: str | None
|
||||
) -> tuple[DropPolicy, ...]:
|
||||
requested: Final = {**_router_defaults(llm_router), **data}
|
||||
router_defaults: Final = _router_defaults(llm_router)
|
||||
model: Final = data.get("model")
|
||||
deployments: Final = (
|
||||
_named_deployments(llm_router, model, team_id) if llm_router is not None and isinstance(model, str) else ()
|
||||
)
|
||||
if not deployments:
|
||||
return (drop_policy(requested), _UNSEEN_DEPLOYMENT)
|
||||
return tuple(drop_policy({**deployment, **requested}) for deployment in deployments)
|
||||
return (drop_policy({**router_defaults, **data}), _UNSEEN_DEPLOYMENT)
|
||||
return tuple(
|
||||
drop_policy(params_sent_to_deployment(deployment, router_defaults, data)) for deployment in deployments
|
||||
)
|
||||
|
||||
|
||||
def resolve_counted_tool_fields(
|
||||
|
|
|
|||
|
|
@ -207,6 +207,7 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
|||
snapshot_tuning_baselines,
|
||||
tuning_limit_violation,
|
||||
)
|
||||
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
|
||||
from litellm.rust_bridge.catalog import Target
|
||||
from litellm.types.utils import (
|
||||
PRICING_OVERRIDES_KEY,
|
||||
|
|
@ -15969,7 +15970,8 @@ def _count_context_management(
|
|||
)
|
||||
)
|
||||
sent_params: Final = tuple(
|
||||
{**params, **router_defaults, **request_litellm_params} for params in deployment_params or ({},)
|
||||
params_sent_to_deployment(params, router_defaults, request_litellm_params)
|
||||
for params in deployment_params or ({},)
|
||||
)
|
||||
if any(context_management_explicitly_dropped(params.get("additional_drop_params")) for params in sent_params):
|
||||
return None
|
||||
|
|
@ -16072,7 +16074,7 @@ def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) ->
|
|||
def _inference_params(
|
||||
litellm_params: Mapping[str, object], responses_request: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
return {**litellm_params, **_router_default_litellm_params(llm_router), **responses_request}
|
||||
return params_sent_to_deployment(litellm_params, _router_default_litellm_params(llm_router), responses_request)
|
||||
|
||||
|
||||
def _count_drop_policy(deployment: Mapping[str, object] | None, responses_request: Mapping[str, object]) -> DropPolicy:
|
||||
|
|
@ -16314,12 +16316,17 @@ async def _count_request_tokens(
|
|||
if _model is not None:
|
||||
model_to_use = _model
|
||||
|
||||
sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params}
|
||||
router_defaults: Final = _router_default_litellm_params(llm_router)
|
||||
sent_params: Final = params_sent_to_deployment(
|
||||
_DEPLOYMENT_LITELLM_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}),
|
||||
router_defaults,
|
||||
request_litellm_params,
|
||||
)
|
||||
count_deployment: Final = _deployment_with_request_drop_settings(deployment, sent_params)
|
||||
count_litellm_params: Final = _DEPLOYMENT_LITELLM_PARAMS.validate_python(
|
||||
(count_deployment or {}).get("litellm_params") or {}
|
||||
)
|
||||
deployment_defaults: Final = {**count_litellm_params, **_router_default_litellm_params(llm_router)}
|
||||
deployment_defaults: Final = params_sent_to_deployment(count_litellm_params, router_defaults, {})
|
||||
conversation_empty: Final = not messages and not contents
|
||||
if prompt and not conversation_empty and not deployment_drops_params(count_deployment):
|
||||
raise ProxyException(
|
||||
|
|
|
|||
|
|
@ -242,6 +242,7 @@ from litellm.router_strategy.least_busy import (
|
|||
picks_reserve,
|
||||
release_started_attempt,
|
||||
)
|
||||
from litellm.router_utils import default_litellm_params as default_params
|
||||
from litellm.router_utils.cooldown_handlers import get_cooldown_deployment_info, mark_deployment_selection_failure
|
||||
from litellm.router_utils.endpoint_eligibility import endpoint_for_call, selected_endpoint
|
||||
from litellm.router_utils.kubernetes_pod_discovery import (
|
||||
|
|
@ -3203,13 +3204,11 @@ class RustCore(
|
|||
)
|
||||
|
||||
def params_sent_to_deployment(self, request: Mapping[str, object], deployment_id: object) -> Mapping[str, object]:
|
||||
"""The request as a deployment received it: its params, the router defaults, its tool choice, then
|
||||
"""The request as a deployment received it: its params, the router defaults it leaves unset, then
|
||||
the request (``_params_sent_to_deployment``)."""
|
||||
record: Final = self._handle.deployment(deployment_id) if isinstance(deployment_id, str) else None
|
||||
params: Final = as_str_mapping((as_str_mapping(record) or {}).get("litellm_params")) or {}
|
||||
defaults: Final = {k: v for k, v in self._resolved.default_litellm_params.items() if v is not None}
|
||||
tool_choice: Final = {"tool_choice": params["tool_choice"]} if params.get("tool_choice") is not None else {}
|
||||
return MappingProxyType({**params, **defaults, **tool_choice, **request})
|
||||
return default_params.params_sent_to_deployment(params, self._resolved.default_litellm_params, request)
|
||||
|
||||
def _drive_sync_steps(self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]") -> Mapping[str, object]:
|
||||
"""Drive a call whose only steps are cache ops (``report_failure``) to its end."""
|
||||
|
|
@ -3434,15 +3433,17 @@ class RustCore(
|
|||
) -> list[str]: # mutable-ok: the step's answer
|
||||
"""The candidates that refuse the audio or videos a Decisions call carries."""
|
||||
group: Final = str(step.get("model_group"))
|
||||
defaults: Final = {k: v for k, v in self._resolved.default_litellm_params.items() if v is not None}
|
||||
sent: Final = {**defaults, **request}
|
||||
audio: Final = bool(request.get("audio"))
|
||||
videos: Final = bool(request.get("videos"))
|
||||
return [
|
||||
str(id)
|
||||
for id in _as_list(step.get("candidates"))
|
||||
if refuses_decisions_media(
|
||||
as_str_mapping(self._handle.deployment(str(id))) or {}, group, sent, audio=audio, videos=videos
|
||||
as_str_mapping(self._handle.deployment(str(id))) or {},
|
||||
group,
|
||||
self.params_sent_to_deployment(request, str(id)),
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ from litellm.router_strategy.complexity_router.context_compaction import (
|
|||
initialize_compaction_state,
|
||||
surface_for_call,
|
||||
)
|
||||
from litellm.router_utils.default_litellm_params import router_defaults_for_deployment
|
||||
from litellm.router_utils.endpoint_eligibility import endpoint_for_call, endpoint_selection
|
||||
from litellm.vector_stores.search_query import store_search_params
|
||||
|
||||
|
|
@ -2176,7 +2177,7 @@ class RustEndpoints:
|
|||
return await self._direct("amoderation", prepared)
|
||||
|
||||
def text_completion(self, request: Mapping[str, object]) -> object:
|
||||
"""The sync `text_completion`: one pick, no retries, the group name as the model."""
|
||||
"""The sync `text_completion`: one pick, no retries."""
|
||||
from litellm.litellm_core_utils.routing_common_utils import (
|
||||
deployment_model_id_pin,
|
||||
metadata_defaults_under_request,
|
||||
|
|
@ -2197,17 +2198,21 @@ class RustEndpoints:
|
|||
request_kwargs=kwargs,
|
||||
)
|
||||
data: Final = as_dict(deployment.get("litellm_params"))
|
||||
for key, value in self._core.resolved.default_litellm_params.items():
|
||||
if key not in kwargs:
|
||||
kwargs[key] = value
|
||||
elif key == "metadata":
|
||||
metadata_bucket(kwargs, key).update(
|
||||
metadata_defaults_under_request(defaults=as_dict(value), request_metadata=as_dict(kwargs[key]))
|
||||
defaults: Final = self._core.resolved.default_litellm_params
|
||||
for key, value in router_defaults_for_deployment(without(defaults, "metadata"), data).items():
|
||||
kwargs.setdefault(key, value)
|
||||
if "metadata" in defaults:
|
||||
metadata_bucket(kwargs, "metadata").update(
|
||||
metadata_defaults_under_request(
|
||||
defaults=as_dict(defaults["metadata"]), request_metadata=as_dict(kwargs["metadata"])
|
||||
)
|
||||
)
|
||||
pinned: Final = deployment_model_id_pin(data)
|
||||
if pinned is not None:
|
||||
kwargs["model_id"] = pinned
|
||||
return self._invoke("text_completion")(**{**data, "prompt": prompt, "caching": self._cache_responses, **kwargs})
|
||||
return self._invoke("text_completion")(
|
||||
**{**data, "prompt": prompt, "caching": self._cache_responses, **without(kwargs, "model")}
|
||||
)
|
||||
|
||||
async def aadapter_completion(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -41,6 +41,11 @@ from litellm.router_strategy.tag_based_routing import (
|
|||
get_tags_from_request_kwargs,
|
||||
tags_with_serving_deployment_tags,
|
||||
)
|
||||
from litellm.router_utils.default_litellm_params import (
|
||||
deployment_sets_litellm_param,
|
||||
params_sent_to_deployment,
|
||||
router_defaults_for_deployment,
|
||||
)
|
||||
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
|
||||
deployment_has_io_token_limits, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values
|
||||
refund_stale_reservation_before_retry,
|
||||
|
|
@ -119,8 +124,7 @@ def _drops_malformed_request_tools(
|
|||
) -> bool:
|
||||
if isinstance(request.get("tools") or [], (list, tuple)) or not deployment_params.get("tools"):
|
||||
return False
|
||||
set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
|
||||
return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools")
|
||||
return drop_policy(params_sent_to_deployment(deployment_params, defaults, request)).drops("tools")
|
||||
|
||||
|
||||
def _malformed_request_params_the_deployment_sets(
|
||||
|
|
@ -136,8 +140,7 @@ def _malformed_request_params_the_deployment_sets(
|
|||
try:
|
||||
validate_litellm_params_strictly(GenericLiteLLMParams, overrides)
|
||||
except ValidationError as error:
|
||||
set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
|
||||
policy: Final = drop_policy({**deployment_params, **set_defaults, **request})
|
||||
policy: Final = drop_policy(params_sent_to_deployment(deployment_params, defaults, request))
|
||||
malformed: Final = (str(detail["loc"][0]) for detail in error.errors() if detail["loc"])
|
||||
return frozenset(key for key in malformed if policy.drops(key))
|
||||
return frozenset()
|
||||
|
|
@ -175,21 +178,13 @@ def merge_tools_from_deployment(
|
|||
)
|
||||
|
||||
|
||||
def _deployment_sets_litellm_param(deployment_params: Mapping[str, object], key: str) -> bool:
|
||||
value: Final = deployment_params.get(key)
|
||||
if value is None:
|
||||
return False
|
||||
field: Final = LiteLLM_Params.model_fields.get(key)
|
||||
return field is None or value != field.default # pyright: ignore[reportAny] # litellm's untyped helpers and request values
|
||||
|
||||
|
||||
def _forwarded_alias_marker_keys_the_deployment_sets(
|
||||
deployment_params: Mapping[str, object], forwarded_keys: object
|
||||
) -> tuple[str, ...]:
|
||||
if not isinstance(forwarded_keys, tuple):
|
||||
return ()
|
||||
keys: Final[tuple[object, ...]] = forwarded_keys # pyright: ignore[reportUnknownVariableType] # the router's own marker
|
||||
return tuple(key for key in keys if isinstance(key, str) and _deployment_sets_litellm_param(deployment_params, key))
|
||||
return tuple(key for key in keys if isinstance(key, str) and deployment_sets_litellm_param(deployment_params, key))
|
||||
|
||||
|
||||
# --- reasoning effort -------------------------------------------------------------------------
|
||||
|
|
@ -355,19 +350,18 @@ def call_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[s
|
|||
|
||||
def update_kwargs_with_default_litellm_params(
|
||||
shaping: Shaping,
|
||||
deployment_params: Mapping[str, object],
|
||||
kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place
|
||||
metadata_variable_name: str,
|
||||
) -> None:
|
||||
"""``_update_kwargs_with_default_litellm_params``: unset non-None defaults fill the request;
|
||||
the default metadata fills the keys the request leaves unset, the router's caching groups excepted."""
|
||||
defaults: Final = dict(without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs))
|
||||
metadata_defaults: Final = defaults.pop("metadata", {}) or {} # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values
|
||||
for key, value in defaults.items():
|
||||
if value is None:
|
||||
continue
|
||||
"""``_update_kwargs_with_default_litellm_params``: the non-None defaults the request leaves unset and
|
||||
the deployment does not set itself fill the request; the default metadata fills the keys the request
|
||||
leaves unset, the router's caching groups excepted."""
|
||||
defaults: Final = without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs)
|
||||
for key, value in router_defaults_for_deployment(without(defaults, "metadata"), deployment_params).items():
|
||||
kwargs.setdefault(key, value)
|
||||
bucket: Final = metadata_bucket(kwargs, metadata_variable_name)
|
||||
bucket.update(metadata_defaults_under_request(defaults=as_dict(metadata_defaults), request_metadata=bucket)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values
|
||||
bucket.update(metadata_defaults_under_request(defaults=as_dict(defaults.get("metadata")), request_metadata=bucket))
|
||||
|
||||
|
||||
# --- clientside credentials -------------------------------------------------------------------
|
||||
|
|
@ -515,11 +509,11 @@ def update_kwargs_with_deployment(
|
|||
)
|
||||
_set_timeouts(shaping, deployment_params, kwargs, function_name, splits_stream_timeout)
|
||||
|
||||
if router_config_requests_stream_usage(kwargs, (deployment_params, defaults)):
|
||||
if router_config_requests_stream_usage(kwargs, (params_sent_to_deployment(deployment_params, defaults, {}),)):
|
||||
kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
|
||||
STREAM_USAGE_INJECTED_FIELD
|
||||
] = True
|
||||
update_kwargs_with_default_litellm_params(shaping, kwargs, metadata_variable_name)
|
||||
update_kwargs_with_default_litellm_params(shaping, deployment_params, kwargs, metadata_variable_name)
|
||||
pinned_model_id: Final = deployment_model_id_pin(deployment_params)
|
||||
if pinned_model_id is not None:
|
||||
kwargs["model_id"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ import traceback
|
|||
import uuid
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
|
@ -46,6 +45,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
|||
count_capability_routers,
|
||||
)
|
||||
from litellm.router_utils.cooldown_handlers import set_cooldown_deployments
|
||||
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
|
||||
from litellm.router_utils.endpoint_eligibility import endpoint_selection
|
||||
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_io_token_refund_failure
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
|
||||
|
|
@ -538,9 +538,7 @@ class RustLegacyMembers(RustRegistryMembers, RustOptionalChecks):
|
|||
"""``_params_sent_to_deployment`` of a ``Deployment``, which ``RoutingContext.deployment_params`` hands in."""
|
||||
params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
|
||||
configured: Final = as_str_mapping(RoutingContext(self).settings().default_litellm_params) # pyright: ignore[reportArgumentType, reportUnknownMemberType, reportUnknownArgumentType] # the core's RoutingContext
|
||||
defaults: Final = {k: v for k, v in (configured or {}).items() if v is not None}
|
||||
tool_choice: Final = {"tool_choice": params["tool_choice"]} if params.get("tool_choice") is not None else {}
|
||||
return MappingProxyType({**params, **defaults, **tool_choice, **request_kwargs})
|
||||
return params_sent_to_deployment(params, configured or {}, request_kwargs)
|
||||
|
||||
|
||||
__all__ = ["RustLegacyMembers"]
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials
|
|||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.router_utils.cooldown_cache import CooldownCacheValue
|
||||
from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback
|
||||
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
|
||||
from litellm.types.llms.anthropic import MESSAGE_THREAD_NOT_FOUND_ERROR_CODE
|
||||
from litellm.types.router import Deployment
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -888,8 +889,9 @@ def is_caller_entra_credential_failure(
|
|||
|
||||
credential_name: Final = deployment.litellm_params.litellm_credential_name
|
||||
configured_by_precedence: Final[tuple[Mapping[str, object], ...]] = (
|
||||
router_defaults,
|
||||
_CREDENTIAL_VALUES_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)),
|
||||
_CREDENTIAL_VALUES_ADAPTER.validate_python(
|
||||
params_sent_to_deployment(deployment.litellm_params.model_dump(exclude_none=True), router_defaults, {})
|
||||
),
|
||||
_CREDENTIAL_VALUES_ADAPTER.validate_python(
|
||||
CredentialAccessor.get_credential_values(credential_name) if credential_name is not None else {}
|
||||
),
|
||||
|
|
|
|||
33
litellm/router_utils/default_litellm_params.py
Normal file
33
litellm/router_utils/default_litellm_params.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
|
||||
def deployment_sets_litellm_param(deployment_params: Mapping[str, object], key: str) -> bool:
|
||||
value: Final = deployment_params.get(key)
|
||||
if value is None:
|
||||
return False
|
||||
field: Final = LiteLLM_Params.model_fields.get(key)
|
||||
return field is None or value != field.default # pyright: ignore[reportAny] # pydantic types FieldInfo.default as Any
|
||||
|
||||
|
||||
def router_defaults_for_deployment(
|
||||
router_defaults: Mapping[str, object], deployment_params: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in router_defaults.items()
|
||||
if value is not None and not deployment_sets_litellm_param(deployment_params, key)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def params_sent_to_deployment(
|
||||
deployment_params: Mapping[str, object], router_defaults: Mapping[str, object], request: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
{**deployment_params, **router_defaults_for_deployment(router_defaults, deployment_params), **request}
|
||||
)
|
||||
|
|
@ -6,6 +6,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -91,7 +92,8 @@ def request_drops_params(
|
|||
request_flag: Final = _explicit_drop_params(kwargs)
|
||||
if request_flag is not None:
|
||||
return request_flag
|
||||
router_flag: Final = _explicit_drop_params(router_defaults)
|
||||
if router_flag is not None:
|
||||
return router_flag
|
||||
return any(_explicit_drop_params(params) is True for params in deployments)
|
||||
if not deployments:
|
||||
return _explicit_drop_params(router_defaults) is True
|
||||
return any(
|
||||
_explicit_drop_params(params_sent_to_deployment(params, router_defaults, {})) is True for params in deployments
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4174,7 +4174,7 @@ async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rat
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
|
||||
async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing_who_takes_clips(
|
||||
async def test_a_deployment_drop_params_beats_a_router_default_when_choosing_who_takes_clips(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
|
|
@ -4194,13 +4194,13 @@ async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing
|
|||
return await response if asyncio.iscoroutine(response) else response
|
||||
|
||||
await send()
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
|
||||
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
|
||||
|
||||
await send(drop_params=True)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
|
||||
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
|
||||
|
||||
await send(drop_params=False)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
|
||||
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it(
|
||||
|
|
|
|||
|
|
@ -6000,6 +6000,28 @@ class TestResponsesInputTokens:
|
|||
assert counter.await_args.kwargs["request"].tools is None
|
||||
assert "tools" not in counter.await_args.kwargs["responses_request"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_drop_params", "expected_status"),
|
||||
[({}, 200), ({"drop_params": False}, 400)],
|
||||
ids=["router-default-drops", "deployment-keeps-over-router-default"],
|
||||
)
|
||||
def test_malformed_tools_follow_the_deployment_drop_params_over_the_router_default(
|
||||
self, deployment_drop_params: dict[str, bool], expected_status: int
|
||||
) -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-5.6",
|
||||
"litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-test", **deployment_drop_params},
|
||||
}
|
||||
],
|
||||
default_litellm_params={"drop_params": True},
|
||||
)
|
||||
|
||||
response, _ = self._post_input_tokens({"model": "gpt-5.6", "input": "hi", "tools": 5}, llm_router=router)
|
||||
|
||||
assert response.status_code == expected_status, response.text
|
||||
|
||||
def test_malformed_tools_for_a_model_outside_the_router_get_a_clean_400(self) -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[{"model_name": "gpt-5.2", "litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test"}}]
|
||||
|
|
|
|||
|
|
@ -1582,7 +1582,10 @@ def _build_log_messages(log_text: JsonValue) -> list[dict[str, JsonValue]]:
|
|||
|
||||
|
||||
async def _count_on_a_model_without_a_count_api(
|
||||
monkeypatch: pytest.MonkeyPatch, body: Mapping[str, object], router_defaults: dict[str, JsonValue] | None = None
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
body: Mapping[str, object],
|
||||
router_defaults: dict[str, JsonValue] | None = None,
|
||||
deployment_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> JSONResponse | dict[str, JsonValue]:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
|
|
@ -1590,7 +1593,7 @@ async def _count_on_a_model_without_a_count_api(
|
|||
model_list=[
|
||||
{
|
||||
"model_name": "mistral-tool-result",
|
||||
"litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "sk-test"},
|
||||
"litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "sk-test", **deployment_params},
|
||||
}
|
||||
],
|
||||
default_litellm_params=router_defaults,
|
||||
|
|
@ -1633,14 +1636,22 @@ async def test_count_tokens_without_a_count_api_counts_a_non_string_tool_result_
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("body_drop_params", "expected_status"), (({}, None), ({"drop_params": False}, 400)))
|
||||
async def test_count_tokens_honors_router_default_drop_params_unless_the_body_overrides_it(
|
||||
monkeypatch: pytest.MonkeyPatch, body_drop_params: dict[str, JsonValue], expected_status: int | None
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_params", "body_drop_params", "expected_status"),
|
||||
(({}, {}, None), ({}, {"drop_params": False}, 400), ({"drop_params": False}, {}, 400)),
|
||||
ids=("router-default", "body-over-router-default", "deployment-over-router-default"),
|
||||
)
|
||||
async def test_count_tokens_honors_router_default_drop_params_unless_the_body_or_deployment_overrides_it(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_params: dict[str, JsonValue],
|
||||
body_drop_params: dict[str, JsonValue],
|
||||
expected_status: int | None,
|
||||
) -> None:
|
||||
response: Final = await _count_on_a_model_without_a_count_api(
|
||||
monkeypatch,
|
||||
{"model": "mistral-tool-result", "messages": _build_log_messages(None), **body_drop_params},
|
||||
router_defaults={"drop_params": True},
|
||||
deployment_params=deployment_params,
|
||||
)
|
||||
|
||||
match response:
|
||||
|
|
@ -2992,6 +3003,7 @@ _NATIVE_RESPONSES_BASE: Final = {
|
|||
_FIREWORKS_DEEPSEEK_V4_1: "https://unused.invalid",
|
||||
_XAI_GROK_4_3: "https://unused.invalid",
|
||||
}
|
||||
_BRIDGED_RESPONSES_BASE: Final = "https://unused.cognitiveservices.azure.com/models"
|
||||
|
||||
|
||||
def _responses_function(tool: Mapping[str, object], **strict: bool | None) -> dict[str, object]:
|
||||
|
|
@ -3300,26 +3312,34 @@ async def test_a_responses_count_that_asks_for_the_chat_completions_bridge_frame
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_responses_count_frames_for_the_api_base_router_defaults_send_inference_to(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_params", "inference_api_base"),
|
||||
[
|
||||
pytest.param({}, _BRIDGED_RESPONSES_BASE, id="router-default-fills-an-unset-api-base"),
|
||||
pytest.param(
|
||||
{"api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]},
|
||||
_NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1],
|
||||
id="deployment-api-base-beats-the-router-default",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_a_responses_count_frames_for_the_api_base_inference_is_sent_to(
|
||||
monkeypatch: pytest.MonkeyPatch, deployment_params: dict[str, object], inference_api_base: str
|
||||
):
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "counted",
|
||||
"litellm_params": {
|
||||
"api_key": "unused",
|
||||
"model": _AZURE_GROK_4_1,
|
||||
"api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1],
|
||||
},
|
||||
"litellm_params": {"api_key": "unused", "model": _AZURE_GROK_4_1, **deployment_params},
|
||||
"model_info": {},
|
||||
}
|
||||
],
|
||||
default_litellm_params={"api_base": "https://unused.cognitiveservices.azure.com/models"},
|
||||
default_litellm_params={"api_base": _BRIDGED_RESPONSES_BASE},
|
||||
)
|
||||
chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False)
|
||||
sent_alone: Final = _single_deployment_router({"model": _AZURE_GROK_4_1, "api_base": inference_api_base}, {})
|
||||
expected: Final = await _count_with_router(monkeypatch, sent_alone, (_GET_WEATHER,), responses=True)
|
||||
|
||||
assert await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=True) == chat_count
|
||||
assert await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=True) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -4157,10 +4177,20 @@ async def test_messages_count_tokens_for_a_model_with_no_deployment_drops_an_inv
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gemini_messages_count_tokens_rejects_an_invalid_edit_without_drop_params() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_params", "router_defaults"),
|
||||
(({}, {}), ({"drop_params": False}, {"drop_params": True})),
|
||||
ids=("no-drop-params", "deployment-over-router-default"),
|
||||
)
|
||||
async def test_gemini_messages_count_tokens_rejects_an_invalid_edit_without_drop_params(
|
||||
deployment_params: dict[str, JsonValue], router_defaults: dict[str, JsonValue]
|
||||
) -> None:
|
||||
with patch.object(litellm, "drop_params", False):
|
||||
response, sent_bodies = await _count_through_the_messages_endpoint(
|
||||
Router(model_list=[_gemini_deployment("gemini-agent", {})]),
|
||||
Router(
|
||||
model_list=[_gemini_deployment("gemini-agent", deployment_params)],
|
||||
default_litellm_params=router_defaults,
|
||||
),
|
||||
{"model": "gemini-agent", "context_management": _INVALID_COMPACT_EDIT},
|
||||
)
|
||||
|
||||
|
|
@ -5480,6 +5510,7 @@ async def _claude_count(
|
|||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_params: Mapping[str, JsonValue],
|
||||
responses_request: Mapping[str, object],
|
||||
router_defaults: dict[str, JsonValue] | None = None,
|
||||
) -> tuple[TokenCountResponse, list[dict[str, JsonValue]]]:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
|
|
@ -5492,7 +5523,8 @@ async def _claude_count(
|
|||
**deployment_params,
|
||||
},
|
||||
}
|
||||
]
|
||||
],
|
||||
default_litellm_params=router_defaults,
|
||||
)
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
|
||||
bodies: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: collects what the mock peer receives
|
||||
|
|
@ -5516,14 +5548,15 @@ async def _claude_count(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_params", "request_params"),
|
||||
[({"drop_params": True}, {}), ({}, {"drop_params": True})],
|
||||
ids=["deployment", "request"],
|
||||
("deployment_params", "request_params", "router_defaults"),
|
||||
[({"drop_params": True}, {}, None), ({}, {"drop_params": True}, None), ({}, {}, {"drop_params": True})],
|
||||
ids=["deployment", "request", "router-default"],
|
||||
)
|
||||
async def test_responses_count_on_claude_downgrades_forced_tool_choice_under_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_params: Mapping[str, JsonValue],
|
||||
request_params: Mapping[str, JsonValue],
|
||||
router_defaults: dict[str, JsonValue] | None,
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_forced_tool_use": False})
|
||||
|
||||
|
|
@ -5531,6 +5564,7 @@ async def test_responses_count_on_claude_downgrades_forced_tool_choice_under_dro
|
|||
monkeypatch,
|
||||
deployment_params,
|
||||
{"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required", **request_params},
|
||||
router_defaults,
|
||||
)
|
||||
|
||||
assert (result.error, result.total_tokens) == (False, 77)
|
||||
|
|
@ -5539,13 +5573,25 @@ async def test_responses_count_on_claude_downgrades_forced_tool_choice_under_dro
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_params", "router_defaults"),
|
||||
[({}, None), ({"drop_params": False}, {"drop_params": True})],
|
||||
ids=["no-drop-params", "deployment-over-router-default"],
|
||||
)
|
||||
async def test_responses_count_on_claude_rejects_forced_tool_choice_without_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_params: Mapping[str, JsonValue],
|
||||
router_defaults: dict[str, JsonValue] | None,
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_forced_tool_use": False})
|
||||
|
||||
with pytest.raises(ProxyException) as rejected:
|
||||
await _claude_count(monkeypatch, {}, {"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required"})
|
||||
await _claude_count(
|
||||
monkeypatch,
|
||||
deployment_params,
|
||||
{"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required"},
|
||||
router_defaults,
|
||||
)
|
||||
|
||||
assert (rejected.value.code, "forced tool use" in rejected.value.message) == ("400", True)
|
||||
|
||||
|
|
|
|||
|
|
@ -1138,7 +1138,7 @@ async def test_jev_bounds_complete_input_without_changing_forwarded_messages(
|
|||
[
|
||||
("false", None, None, None, "truncated"),
|
||||
("true", None, None, None, "within_budget"),
|
||||
("true", "false", None, None, "truncated"),
|
||||
("true", "false", None, None, "within_budget"),
|
||||
("true", None, 8192, None, "truncated"),
|
||||
("true", None, 8192, "default", "truncated"),
|
||||
("true", None, None, "default", "truncated"),
|
||||
|
|
@ -1199,7 +1199,7 @@ async def test_jev_budget_uses_effective_headers_and_every_deployment(
|
|||
assert telemetry["classifier_input_tokens"] <= telemetry["classifier_input_budget_tokens"]
|
||||
sent: Final = JevSystemOneRequest.model_validate_json(route.calls[0].request.content)
|
||||
if other_member is None:
|
||||
assert route.calls[0].request.headers[header] == (default_header or mode)
|
||||
assert route.calls[0].request.headers[header] == mode
|
||||
if explicit_cap is not None:
|
||||
assert telemetry["classifier_input_budget_tokens"] < explicit_cap
|
||||
assert (ask in sent.state) is (expected_status == "within_budget")
|
||||
|
|
|
|||
34
tests/unit/router_utils/test_default_litellm_params.py
Normal file
34
tests/unit/router_utils/test_default_litellm_params.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
|
||||
|
||||
_ROUTER_DEFAULTS: Final = {"vertex_location": "us-east5", "use_in_pass_through": True, "timeout": None}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment", "request_params", "sent"),
|
||||
[
|
||||
({"vertex_location": "eu"}, {}, {"vertex_location": "eu", "use_in_pass_through": True}),
|
||||
({}, {}, {"vertex_location": "us-east5", "use_in_pass_through": True}),
|
||||
({"vertex_location": None}, {}, {"vertex_location": "us-east5", "use_in_pass_through": True}),
|
||||
({"vertex_location": "eu"}, {"vertex_location": "asia-northeast1"}, {"vertex_location": "asia-northeast1"}),
|
||||
({"use_in_pass_through": False}, {}, {"vertex_location": "us-east5", "use_in_pass_through": True}),
|
||||
],
|
||||
ids=[
|
||||
"deployment-over-router-default",
|
||||
"router-default-fills-unset",
|
||||
"router-default-fills-none",
|
||||
"request-over-both",
|
||||
"deployment-field-default-counts-as-unset",
|
||||
],
|
||||
)
|
||||
def test_params_sent_to_deployment_prefer_the_request_then_the_deployment_then_the_router_default(
|
||||
deployment: Mapping[str, object], request_params: Mapping[str, object], sent: Mapping[str, object]
|
||||
) -> None:
|
||||
merged: Final = params_sent_to_deployment(deployment, _ROUTER_DEFAULTS, request_params)
|
||||
|
||||
assert {key: merged[key] for key in sent} == sent
|
||||
assert "timeout" not in merged
|
||||
|
|
@ -55,7 +55,7 @@ def test_the_drop_params_decision_is_consulted_only_for_a_non_integer_priority()
|
|||
assert resolve_request_priority("1", 3, drops_params=_never) == 1
|
||||
|
||||
|
||||
def test_the_global_drop_params_wins_then_the_request_then_the_router_defaults_then_the_deployments(
|
||||
def test_the_global_drop_params_wins_then_the_request_then_each_deployment_over_the_router_defaults(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
|
|
@ -67,7 +67,10 @@ def test_the_global_drop_params_wins_then_the_request_then_the_router_defaults_t
|
|||
assert request_drops_params({}, {}, ({"drop_params": True},)) is True
|
||||
assert request_drops_params({}, {}, ({"drop_params": True}, {"model": "openai/gpt-5.4-nano"})) is True
|
||||
assert request_drops_params({}, {}, ({"drop_params": False}, {"model": "openai/gpt-5.4-nano"})) is False
|
||||
assert request_drops_params({}, {"drop_params": False}, ({"drop_params": True},)) is False
|
||||
assert request_drops_params({}, {"drop_params": False}, ({"drop_params": True},)) is True
|
||||
assert request_drops_params({}, {"drop_params": True}, ({"drop_params": False},)) is False
|
||||
one_keeps_one_inherits: Final = ({"drop_params": False}, {"model": "openai/gpt-5.4-nano"})
|
||||
assert request_drops_params({}, {"drop_params": True}, one_keeps_one_inherits) is True
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
assert request_drops_params({}, {}, ()) is True
|
||||
assert request_drops_params({}, {}, ({"model": "openai/gpt-5.4-nano"},)) is True
|
||||
|
|
|
|||
|
|
@ -216,18 +216,17 @@ async def test_sync_text_completion_keeps_the_request_metadata_over_the_default_
|
|||
assert _bucket(_sent(host))["requester_metadata"] == {"env": "dev", "region": "us", "team": "t1"}
|
||||
|
||||
|
||||
async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model(
|
||||
async def test_sync_text_completion_sends_the_deployment_model_not_the_model_group_name(
|
||||
make_core: MakeCore, host: EndpointHost
|
||||
):
|
||||
"""The request's model="g" is spread after the deployment params, so it replaces "openai/d1"."""
|
||||
core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
|
||||
core.text_completion(model="g", prompt="p")
|
||||
call: Final = host.calls[0]
|
||||
assert (call.model, call.kwargs["api_base"]) == ("g", "https://d1.example")
|
||||
assert (call.model, call.kwargs["api_base"]) == ("openai/d1", "https://d1.example")
|
||||
|
||||
|
||||
async def test_sync_text_completion_is_not_retried(make_core: MakeCore, host: EndpointHost):
|
||||
host.answer("g", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
|
||||
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3)
|
||||
with pytest.raises(litellm.InternalServerError):
|
||||
core.text_completion(model="g", prompt="p")
|
||||
|
|
|
|||
|
|
@ -640,7 +640,7 @@ def test_deployment_params_merge_in_the_order_the_deployment_receives_them(make_
|
|||
params: Final = RoutingContext(core).deployment_params({"top_p": 0.7}, target) # pyright: ignore[reportArgumentType] # the core a context wraps
|
||||
|
||||
assert params["model"] == "openai/a"
|
||||
assert params["temperature"] == 0.9
|
||||
assert params["temperature"] == 0.1
|
||||
assert params["tool_choice"] == "auto"
|
||||
assert params["top_p"] == 0.7
|
||||
assert RoutingContext(core).deployment_params({}, None).get("temperature") == 0.9 # pyright: ignore[reportArgumentType] # see above
|
||||
|
|
|
|||
|
|
@ -15107,7 +15107,21 @@ async def test_router_caller_entra_field_that_changes_nothing_still_cools_down_a
|
|||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
async def test_router_caller_repeating_the_secret_an_empty_router_default_selects_still_cools_down(monkeypatch):
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_secret", "router_default_secret", "caller_secret"),
|
||||
[
|
||||
pytest.param({}, "", "environment-secret", id="empty-router-default-selects-the-environment"),
|
||||
pytest.param(
|
||||
{"client_secret": "configured-secret"},
|
||||
"router-default-secret",
|
||||
"configured-secret",
|
||||
id="deployment-secret-beats-the-router-default",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_router_caller_repeating_the_configured_secret_still_cools_down(
|
||||
monkeypatch, deployment_secret, router_default_secret, caller_secret
|
||||
):
|
||||
monkeypatch.setenv("AZURE_CLIENT_SECRET", "environment-secret")
|
||||
router = Router(
|
||||
model_list=[
|
||||
|
|
@ -15119,12 +15133,12 @@ async def test_router_caller_repeating_the_secret_an_empty_router_default_select
|
|||
"api_version": "2025-04-01-preview",
|
||||
"tenant_id": "configured-tenant",
|
||||
"client_id": "configured-client",
|
||||
"client_secret": "configured-secret",
|
||||
**deployment_secret,
|
||||
},
|
||||
"model_info": {"id": "configured-deployment"},
|
||||
}
|
||||
],
|
||||
default_litellm_params={"client_secret": ""},
|
||||
default_litellm_params={"client_secret": router_default_secret},
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.AuthenticationError):
|
||||
|
|
@ -15138,7 +15152,7 @@ async def test_router_caller_repeating_the_secret_an_empty_router_default_select
|
|||
model="gpt-4.1",
|
||||
)
|
||||
),
|
||||
client_secret="environment-secret",
|
||||
client_secret=caller_secret,
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.RouterRateLimitError) as excinfo:
|
||||
|
|
@ -20514,3 +20528,161 @@ def test_registry_finds_a_team_deployment_by_public_and_internal_name_for_its_te
|
|||
assert [row["model_info"]["id"] for row in registry.deployments(internal_name, team_id="team-a")] == ["team-row"]
|
||||
assert list(registry.deployments("public-chat", team_id="team-b")) == []
|
||||
assert list(registry.deployments("public-chat")) == []
|
||||
|
||||
|
||||
_ROUTER_DEFAULT_REGION: Final = "us-east-1"
|
||||
_BEDROCK_URL: Final = r"https://bedrock-runtime\.[a-z0-9-]+\.amazonaws\.com/model/.+"
|
||||
_BEDROCK_CONVERSE_REPLY: Final = {
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
|
||||
}
|
||||
_REGION_PRECEDENCE: Final = pytest.mark.parametrize(
|
||||
("deployment_region", "request_region", "called_region"),
|
||||
[
|
||||
("eu-west-1", None, "eu-west-1"),
|
||||
(None, None, _ROUTER_DEFAULT_REGION),
|
||||
("eu-west-1", "ap-northeast-1", "ap-northeast-1"),
|
||||
],
|
||||
ids=["deployment-over-router-default", "router-default-fills-unset", "request-over-both"],
|
||||
)
|
||||
|
||||
|
||||
def _bedrock_router_with_a_default_region(model: str, deployment_region: str | None) -> Router:
|
||||
region: Final = {} if deployment_region is None else {"aws_region_name": deployment_region}
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "regional",
|
||||
"litellm_params": {
|
||||
"model": model,
|
||||
"aws_access_key_id": "unit-access-key",
|
||||
"aws_secret_access_key": "unit-secret-key",
|
||||
**region,
|
||||
},
|
||||
}
|
||||
],
|
||||
default_litellm_params={"aws_region_name": _ROUTER_DEFAULT_REGION},
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
|
||||
def _request_region(request_region: str | None) -> Mapping[str, str]:
|
||||
return {} if request_region is None else {"aws_region_name": request_region}
|
||||
|
||||
|
||||
def _called_regions(route: respx.Route) -> list[str]:
|
||||
return [call.request.url.host.split(".")[1] for call in route.calls]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_router_httpx_transport")
|
||||
@_REGION_PRECEDENCE
|
||||
async def test_acompletion_calls_the_deployment_region_over_a_conflicting_router_default(
|
||||
deployment_region: str | None, request_region: str | None, called_region: str
|
||||
) -> None:
|
||||
router: Final = _bedrock_router_with_a_default_region(
|
||||
"bedrock/converse/anthropic.claude-haiku-5-5", deployment_region
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
route: Final = upstream.post(url__regex=_BEDROCK_URL).respond(json=_BEDROCK_CONVERSE_REPLY)
|
||||
await router.acompletion(
|
||||
model="regional", messages=[{"role": "user", "content": "Say hi"}], **_request_region(request_region)
|
||||
)
|
||||
|
||||
assert _called_regions(route) == [called_region]
|
||||
|
||||
|
||||
@_REGION_PRECEDENCE
|
||||
def test_text_completion_calls_the_deployment_region_over_a_conflicting_router_default(
|
||||
deployment_region: str | None, request_region: str | None, called_region: str
|
||||
) -> None:
|
||||
router: Final = _bedrock_router_with_a_default_region(
|
||||
"bedrock/converse/anthropic.claude-haiku-5-5", deployment_region
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
route: Final = upstream.post(url__regex=_BEDROCK_URL).respond(json=_BEDROCK_CONVERSE_REPLY)
|
||||
router.text_completion(model="regional", prompt="Say hi", **_request_region(request_region))
|
||||
|
||||
assert _called_regions(route) == [called_region]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_router_httpx_transport")
|
||||
@_REGION_PRECEDENCE
|
||||
async def test_aembedding_calls_the_deployment_region_over_a_conflicting_router_default(
|
||||
deployment_region: str | None, request_region: str | None, called_region: str
|
||||
) -> None:
|
||||
router: Final = _bedrock_router_with_a_default_region("bedrock/amazon.titan-embed-text-v2:0", deployment_region)
|
||||
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
route: Final = upstream.post(url__regex=_BEDROCK_URL).respond(
|
||||
json={"embedding": [0.1, 0.2], "inputTextTokenCount": 2}
|
||||
)
|
||||
await router.aembedding(model="regional", input=["Say hi"], **_request_region(request_region))
|
||||
|
||||
assert _called_regions(route) == [called_region]
|
||||
|
||||
|
||||
def test_text_completion_calls_the_provider_with_the_deployment_model_not_the_group_name() -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "text-group",
|
||||
"litellm_params": {
|
||||
"model": "text-completion-openai/unit-text-model",
|
||||
"api_base": "https://sync-text-group.invalid/v1",
|
||||
"api_key": "unit-provider-key",
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
provider: Final = upstream.post("https://sync-text-group.invalid/v1/completions").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "cmpl-unit",
|
||||
"object": "text_completion",
|
||||
"created": 1,
|
||||
"model": "unit-text-model",
|
||||
"choices": [{"text": "provider called", "index": 0, "finish_reason": "stop", "logprobs": None}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
response: Final = router.text_completion(model="text-group", prompt="Say hi")
|
||||
|
||||
assert response.choices[0].text == "provider called"
|
||||
assert json.loads(provider.calls.last.request.content)["model"] == "unit-text-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_router_httpx_transport")
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_params", "provider_hits"),
|
||||
[({"max_retries": 1}, 2), ({"max_retries": 3}, 4)],
|
||||
ids=["deployment-max-retries-1", "deployment-max-retries-3"],
|
||||
)
|
||||
async def test_aembedding_retries_at_the_provider_as_often_as_the_deployment_sets(
|
||||
deployment_params: Mapping[str, int], provider_hits: int
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "embed",
|
||||
"litellm_params": {"model": "openai/text-embedding-3-large", "api_key": "sk-test", **deployment_params},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
route: Final = upstream.post("https://api.openai.com/v1/embeddings").respond(
|
||||
500, headers={"retry-after-ms": "1"}, json={"error": {"message": "overloaded", "type": "server_error"}}
|
||||
)
|
||||
with pytest.raises(litellm.InternalServerError):
|
||||
await router.aembedding(model="embed", input=["Say hi"])
|
||||
|
||||
assert len(route.calls) == provider_hits
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue