mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge ef61ca6ee8 into cedae29cc7
This commit is contained in:
commit
c98bf12656
7 changed files with 1049 additions and 4 deletions
|
|
@ -72,6 +72,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
|
|||
"model_group_alias",
|
||||
"enable_weighted_failover",
|
||||
"enable_mid_stream_fallback_continuation",
|
||||
"enable_cross_model_group_collision_check",
|
||||
"enable_tag_filtering",
|
||||
"tag_routing_prefix",
|
||||
"optional_pre_call_checks",
|
||||
|
|
|
|||
|
|
@ -212,6 +212,7 @@ from litellm.router_utils.clientside_credential_handler import (
|
|||
)
|
||||
from litellm.router_utils.common_utils import (
|
||||
MID_STREAM_CONTINUATION_KWARG,
|
||||
RawModelGroupCollisionError,
|
||||
deployments_the_caller_may_use,
|
||||
filter_team_based_models,
|
||||
filter_web_search_deployments,
|
||||
|
|
@ -220,6 +221,7 @@ from litellm.router_utils.common_utils import (
|
|||
get_request_team_id,
|
||||
is_proxy_admin_request,
|
||||
provider_for_generic_call,
|
||||
refuse_litellm_model_spanning_model_groups,
|
||||
request_user_api_key_auth,
|
||||
resolve_model_group_alias,
|
||||
team_may_use_deployment,
|
||||
|
|
@ -1272,6 +1274,7 @@ class Router:
|
|||
fallback_budget_check: FallbackBudgetCheck | None = None,
|
||||
auto_router_capability_limit: AutoRouterCapabilityLimit | None = None,
|
||||
session_affinity_repin_on_failover: bool = False,
|
||||
enable_cross_model_group_collision_check: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the Router class with the given parameters for caching, reliability, and routing strategy.
|
||||
|
|
@ -1315,6 +1318,7 @@ class Router:
|
|||
enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False.
|
||||
fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted).
|
||||
fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback).
|
||||
enable_cross_model_group_collision_check (bool): When True, a request whose model matches no model group but is the `litellm_params.model` of deployments in more than one model group is refused with a 400 instead of being load balanced across all of those groups. Defaults to False.
|
||||
Returns:
|
||||
Router: An instance of the litellm.Router class.
|
||||
|
||||
|
|
@ -1526,6 +1530,7 @@ class Router:
|
|||
self.enable_health_check_routing = enable_health_check_routing
|
||||
self.enable_weighted_failover = enable_weighted_failover
|
||||
self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enable_mid_stream_fallback_continuation)
|
||||
self.enable_cross_model_group_collision_check = self._is_flag_enabled(enable_cross_model_group_collision_check)
|
||||
self.health_check_ignore_transient_errors = health_check_ignore_transient_errors
|
||||
self.background_health_check_model_groups: frozenset[str] | None = (
|
||||
frozenset(background_health_check_model_groups)
|
||||
|
|
@ -8567,6 +8572,7 @@ class Router:
|
|||
or original_model_group is None
|
||||
or is_guardrail_intervention(e)
|
||||
or unsupported_params_without_fallback
|
||||
or isinstance(e, RawModelGroupCollisionError)
|
||||
):
|
||||
raise e
|
||||
|
||||
|
|
@ -13996,6 +14002,7 @@ class Router:
|
|||
"model_group_alias",
|
||||
"enable_weighted_failover",
|
||||
"enable_mid_stream_fallback_continuation",
|
||||
"enable_cross_model_group_collision_check",
|
||||
"enable_tag_filtering",
|
||||
"tag_routing_prefix",
|
||||
"fallback_on_unsupported_params",
|
||||
|
|
@ -14053,9 +14060,13 @@ class Router:
|
|||
elif var == "optional_pre_call_checks":
|
||||
self.set_optional_pre_call_checks(kwargs[var])
|
||||
elif var == "enable_mid_stream_fallback_continuation":
|
||||
self._set_mid_stream_fallback_continuation(kwargs[var])
|
||||
self._set_mid_stream_fallback_continuation(cast(object, kwargs[var])) # cast-ok: untyped **kwargs
|
||||
elif var == "enable_cross_model_group_collision_check":
|
||||
self.enable_cross_model_group_collision_check = self._is_flag_enabled(
|
||||
cast(object, kwargs[var]) # cast-ok: untyped **kwargs
|
||||
)
|
||||
elif var == "retry_policy":
|
||||
self._set_retry_policy(kwargs[var])
|
||||
self._set_retry_policy(cast(object, kwargs[var])) # cast-ok: untyped **kwargs
|
||||
else:
|
||||
value = kwargs[var]
|
||||
# only run routing strategy init if it has changed
|
||||
|
|
@ -14746,7 +14757,7 @@ class Router:
|
|||
request_team_id: Final = get_request_team_id(request_kwargs)
|
||||
# check if aliases set on litellm model alias map
|
||||
if specific_deployment is True:
|
||||
return model, self._drop_strategy_markers(
|
||||
specific_deployments: Final = self._drop_strategy_markers(
|
||||
model,
|
||||
self._filter_reserved_deployments(
|
||||
model=model,
|
||||
|
|
@ -14754,6 +14765,11 @@ class Router:
|
|||
request_team_id=request_team_id,
|
||||
),
|
||||
)
|
||||
if self.enable_cross_model_group_collision_check:
|
||||
refuse_litellm_model_spanning_model_groups(
|
||||
model, deployments_the_caller_may_use(specific_deployments, request_kwargs)
|
||||
)
|
||||
return model, specific_deployments
|
||||
elif model not in self.model_names and self.has_model_id(model):
|
||||
deployment: Final = self.get_deployment(model_id=model)
|
||||
if deployment is not None:
|
||||
|
|
@ -14846,6 +14862,10 @@ class Router:
|
|||
),
|
||||
request_team_id=request_team_id,
|
||||
)
|
||||
if self.enable_cross_model_group_collision_check:
|
||||
refuse_litellm_model_spanning_model_groups(
|
||||
model, deployments_the_caller_may_use(healthy_deployments, request_kwargs)
|
||||
)
|
||||
# If the litellm-model lookup produced candidates that access-group
|
||||
# filtering then removed, treat this the same as the by-name path
|
||||
# being emptied: prevent default-model fallback from bypassing the
|
||||
|
|
|
|||
|
|
@ -330,6 +330,42 @@ def filter_team_based_models(
|
|||
]
|
||||
|
||||
|
||||
def _model_group_of(deployment: Mapping[str, object]) -> tuple[object, object]:
|
||||
"""A team deployment's model group is its team and public name, with no name for a team deployment created
|
||||
before public names; any other deployment's is its `model_name`."""
|
||||
model_info: Final = _OBJECT_MAPPING_ADAPTER.validate_python(deployment.get("model_info") or {})
|
||||
team_id: Final = model_info.get("team_id")
|
||||
public_name: Final = model_info.get("team_public_model_name")
|
||||
if isinstance(public_name, str) and public_name:
|
||||
return team_id, public_name
|
||||
model_name: Final = deployment["model_name"]
|
||||
if isinstance(team_id, str) and isinstance(model_name, str) and model_name.startswith(f"model_name_{team_id}_"):
|
||||
return team_id, None
|
||||
return None, model_name
|
||||
|
||||
|
||||
class RawModelGroupCollisionError(BadRequestError):
|
||||
"""The 400 for a raw `litellm_params.model` that several model groups share; router fallbacks never serve it."""
|
||||
|
||||
|
||||
def refuse_litellm_model_spanning_model_groups(model: str, deployments: Sequence[Mapping[str, object]]) -> None:
|
||||
"""Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group.
|
||||
|
||||
A team with a matching deployment created before public names counts as one model group.
|
||||
"""
|
||||
groups: Final = frozenset(_model_group_of(deployment) for deployment in deployments)
|
||||
unnamed_teams: Final = frozenset(team for team, name in groups if team is not None and name is None)
|
||||
counted: Final = frozenset((team, None) if team in unnamed_teams else (team, name) for team, name in groups)
|
||||
if len(counted) < 2:
|
||||
return
|
||||
raise RawModelGroupCollisionError(
|
||||
message=f"LiteLLM Router: model={model} is the litellm_params.model of more than one model group. "
|
||||
"Send one of the model group names instead",
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
||||
|
||||
def deployments_the_caller_may_use(
|
||||
deployments: Sequence[Mapping[str, object]], request_kwargs: Mapping[str, object] | None
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
|
|
|
|||
|
|
@ -272,4 +272,13 @@ ROUTER_SETTINGS_FIELDS: Final[list[RouterSettingsField]] = [
|
|||
field_default=None,
|
||||
ui_field_name="Disable Cooldowns",
|
||||
),
|
||||
RouterSettingsField(
|
||||
field_name="enable_cross_model_group_collision_check",
|
||||
field_type="Boolean",
|
||||
field_value=None,
|
||||
field_description="Refuse a request for a litellm_params.model that deployments in more than one model "
|
||||
"group share, instead of load balancing it across all of those groups",
|
||||
field_default=False,
|
||||
ui_field_name="Refuse Raw Models Shared Across Groups",
|
||||
),
|
||||
]
|
||||
|
|
|
|||
637
tests/integration/routing/test_raw_model_group_collision.py
Normal file
637
tests/integration/routing/test_raw_model_group_collision.py
Normal file
|
|
@ -0,0 +1,637 @@
|
|||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable, Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import anthropic
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import (
|
||||
Gateway,
|
||||
Scenario,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(240)
|
||||
|
||||
SHARED_MODEL: Final = "openai/integration-shared-raw-model"
|
||||
SOLO_MODEL: Final = "openai/integration-solo-raw-model"
|
||||
DEPLOYMENTS: Final = (
|
||||
("collision-chain-a", "collision-a-1", SHARED_MODEL),
|
||||
("collision-chain-a", "collision-a-2", SHARED_MODEL),
|
||||
("collision-chain-b", "collision-b-1", SHARED_MODEL),
|
||||
("collision-solo", "collision-solo-1", SOLO_MODEL),
|
||||
)
|
||||
CHAIN_A: Final = frozenset({"collision-a-1", "collision-a-2"})
|
||||
TEAM_MODEL: Final = "openai/integration-team-raw-model"
|
||||
TEAM_DEPLOYMENTS: Final = frozenset({"collision-team-1", "collision-team-2"})
|
||||
TEAM_BEFORE_PUBLIC_NAMES: Final = "collision-team-before-public-names"
|
||||
ALIAS: Final = "collision-alias"
|
||||
POOLED_CALLS: Final = 48
|
||||
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
CATCH_ALL_FALLBACK: Final[dict[str, JsonValue]] = {"fallbacks": [{"*": ["collision-solo"]}]}
|
||||
|
||||
Build = Callable[[str, str], tuple[str, dict[str, JsonValue]]]
|
||||
SdkCall = Callable[[Gateway, str], None]
|
||||
BURST: Final = 30
|
||||
|
||||
|
||||
def _json_reply(body: dict[str, JsonValue]) -> Reply:
|
||||
return Reply(body=json.dumps(body).encode())
|
||||
|
||||
|
||||
def _reply(request: Request) -> Reply:
|
||||
if request.target.endswith("/chat/completions"):
|
||||
return _json_reply(
|
||||
{
|
||||
"id": f"chatcmpl-{uuid.uuid4().hex}",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "integration-shared-raw-model",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
|
||||
}
|
||||
)
|
||||
if request.target.endswith("/responses"):
|
||||
return _json_reply(
|
||||
{
|
||||
"id": f"resp_{uuid.uuid4().hex}",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "integration-shared-raw-model",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": f"msg_{uuid.uuid4().hex}",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4},
|
||||
}
|
||||
)
|
||||
if request.target.endswith("/embeddings"):
|
||||
return _json_reply(
|
||||
{
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
|
||||
"model": "integration-shared-raw-model",
|
||||
"usage": {"prompt_tokens": 3, "total_tokens": 3},
|
||||
}
|
||||
)
|
||||
return Reply(status=404)
|
||||
|
||||
|
||||
def _served_by(requests: tuple[Request, ...]) -> tuple[str, ...]:
|
||||
return tuple(request.target.split("/")[1] for request in requests)
|
||||
|
||||
|
||||
def _deployment(wire: Wire, group: str, identity: str, model: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model_name": group,
|
||||
"litellm_params": {"model": model, "api_base": f"{wire.url}/{identity}/v1", "api_key": "synthetic-key"},
|
||||
"model_info": {"id": identity},
|
||||
}
|
||||
|
||||
|
||||
def _team_deployment_from_before_public_names(wire: Wire, identity: str, model: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model_name": f"model_name_{TEAM_BEFORE_PUBLIC_NAMES}_{identity}",
|
||||
"litellm_params": {"model": model, "api_base": f"{wire.url}/{identity}/v1", "api_key": "synthetic-key"},
|
||||
"model_info": {"id": identity, "team_id": TEAM_BEFORE_PUBLIC_NAMES},
|
||||
}
|
||||
|
||||
|
||||
def _config(
|
||||
wire: Wire,
|
||||
directory: Path,
|
||||
router_settings: dict[str, JsonValue],
|
||||
extra_deployments: tuple[tuple[str, str, str], ...] = (),
|
||||
team_deployments: tuple[dict[str, JsonValue], ...] = (),
|
||||
) -> Path:
|
||||
config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
config["model_list"] = [
|
||||
*(_deployment(wire, *deployment) for deployment in (*DEPLOYMENTS, *extra_deployments)),
|
||||
*team_deployments,
|
||||
]
|
||||
config["router_settings"] = {**JSON_OBJECT.validate_python(config["router_settings"]), **router_settings}
|
||||
path: Final = directory / "raw-model-group-collision.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def peer() -> Iterator[Wire]:
|
||||
with wire_server(_reply) as wire:
|
||||
yield wire
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def guarded(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("raw-model-collision-guarded")
|
||||
path: Final = _config(peer, directory, {"enable_cross_model_group_collision_check": True, **CATCH_ALL_FALLBACK})
|
||||
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def unguarded(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("raw-model-collision-default")
|
||||
path: Final = _config(peer, directory, {})
|
||||
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def guarded_without_specific_deployment(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("raw-model-collision-wildcard")
|
||||
path: Final = _config(
|
||||
peer,
|
||||
directory,
|
||||
{
|
||||
"enable_cross_model_group_collision_check": True,
|
||||
"model_group_alias": {ALIAS: SHARED_MODEL},
|
||||
**CATCH_ALL_FALLBACK,
|
||||
},
|
||||
(("collision-wild/*", "collision-wild-1", "openai/*"),),
|
||||
)
|
||||
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def guarded_with_team_models_from_before_public_names(
|
||||
peer: Wire, tmp_path_factory: pytest.TempPathFactory
|
||||
) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("raw-model-collision-before-public-names")
|
||||
path: Final = _config(
|
||||
peer,
|
||||
directory,
|
||||
{"enable_cross_model_group_collision_check": True},
|
||||
team_deployments=(
|
||||
*(_team_deployment_from_before_public_names(peer, identity, TEAM_MODEL) for identity in TEAM_DEPLOYMENTS),
|
||||
_team_deployment_from_before_public_names(peer, "collision-team-solo-1", SOLO_MODEL),
|
||||
),
|
||||
)
|
||||
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def toggled(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("raw-model-collision-toggled")
|
||||
path: Final = _config(peer, directory, {})
|
||||
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
def _chat(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
return "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": marker}]}
|
||||
|
||||
|
||||
def _messages(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
return "/v1/messages", {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}]}
|
||||
|
||||
|
||||
def _responses(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
return "/v1/responses", {"model": model, "input": marker}
|
||||
|
||||
|
||||
def _embeddings(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
return "/v1/embeddings", {"model": model, "input": [marker]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"build",
|
||||
(
|
||||
pytest.param(_chat, id="chat-completions"),
|
||||
pytest.param(_messages, id="anthropic-messages"),
|
||||
pytest.param(_responses, id="responses"),
|
||||
pytest.param(_embeddings, id="embeddings"),
|
||||
),
|
||||
)
|
||||
def test_a_raw_model_string_shared_by_two_groups_is_refused_before_any_upstream_call(
|
||||
guarded: Gateway, peer: Wire, build: Build
|
||||
) -> None:
|
||||
with guarded.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
refused_marker: Final = f"collision-refused-{uuid.uuid4().hex}"
|
||||
control_marker: Final = f"collision-control-{uuid.uuid4().hex}"
|
||||
peer.drain()
|
||||
refused: Final = guarded.request("POST", *build(SHARED_MODEL, refused_marker), key=key)
|
||||
control: Final = guarded.request("POST", *build("collision-chain-b", control_marker), key=key)
|
||||
received: Final = peer.drain()
|
||||
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert SHARED_MODEL in refused.text
|
||||
assert control.status_code == 200, control.text
|
||||
assert _served_by(received) == ("collision-b-1",)
|
||||
assert control_marker in received[0].body.decode()
|
||||
assert refused_marker not in received[0].body.decode()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", (pytest.param(SHARED_MODEL, id="raw-model"), pytest.param(ALIAS, id="alias")))
|
||||
def test_the_check_also_refuses_when_a_wildcard_route_sends_the_request_down_the_generic_lookup(
|
||||
guarded_without_specific_deployment: Gateway, peer: Wire, model: str
|
||||
) -> None:
|
||||
gateway: Final = guarded_without_specific_deployment
|
||||
with gateway.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
peer.drain()
|
||||
refused: Final = gateway.request("POST", *_chat(model, uuid.uuid4().hex), key=key)
|
||||
control: Final = gateway.request("POST", *_chat("collision-chain-b", uuid.uuid4().hex), key=key)
|
||||
received: Final = peer.drain()
|
||||
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert SHARED_MODEL in refused.text
|
||||
assert control.status_code == 200, control.text
|
||||
assert _served_by(received) == ("collision-b-1",)
|
||||
|
||||
|
||||
def _streaming(build: Build) -> Build:
|
||||
def streamed(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
path, body = build(model, marker)
|
||||
return path, {**body, "stream": True}
|
||||
|
||||
return streamed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"build",
|
||||
(
|
||||
pytest.param(_streaming(_chat), id="chat-completions-stream"),
|
||||
pytest.param(_streaming(_messages), id="anthropic-messages-stream"),
|
||||
pytest.param(_streaming(_responses), id="responses-stream"),
|
||||
),
|
||||
)
|
||||
def test_a_streaming_request_for_a_shared_raw_model_string_is_refused_before_any_upstream_call(
|
||||
guarded: Gateway, peer: Wire, build: Build
|
||||
) -> None:
|
||||
with guarded.scenario() as scenario:
|
||||
peer.drain()
|
||||
refused: Final = guarded.request("POST", *build(SHARED_MODEL, uuid.uuid4().hex), key=scenario.key())
|
||||
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert SHARED_MODEL in refused.text
|
||||
assert peer.drain() == ()
|
||||
|
||||
|
||||
def _v1(gateway: Gateway) -> str:
|
||||
return f"{str(gateway.client.base_url).rstrip('/')}/v1"
|
||||
|
||||
|
||||
def _origin(gateway: Gateway) -> str:
|
||||
return str(gateway.client.base_url).rstrip("/")
|
||||
|
||||
|
||||
def _openai_chat(gateway: Gateway, key: str) -> None:
|
||||
with (
|
||||
httpx.Client(timeout=15, trust_env=False) as transport,
|
||||
openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
|
||||
):
|
||||
client.chat.completions.create(model=SHARED_MODEL, messages=[{"role": "user", "content": "sdk"}])
|
||||
|
||||
|
||||
def _openai_responses(gateway: Gateway, key: str) -> None:
|
||||
with (
|
||||
httpx.Client(timeout=15, trust_env=False) as transport,
|
||||
openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
|
||||
):
|
||||
client.responses.create(model=SHARED_MODEL, input="sdk")
|
||||
|
||||
|
||||
def _openai_embeddings(gateway: Gateway, key: str) -> None:
|
||||
with (
|
||||
httpx.Client(timeout=15, trust_env=False) as transport,
|
||||
openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
|
||||
):
|
||||
client.embeddings.create(model=SHARED_MODEL, input=["sdk"])
|
||||
|
||||
|
||||
def _run(call: Callable[[], Awaitable[object]]) -> None:
|
||||
asyncio.run(call())
|
||||
|
||||
|
||||
def _async_openai_chat(gateway: Gateway, key: str) -> None:
|
||||
async def call() -> object:
|
||||
async with (
|
||||
httpx.AsyncClient(timeout=15, trust_env=False) as transport,
|
||||
openai.AsyncOpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
|
||||
):
|
||||
return await client.chat.completions.create(
|
||||
model=SHARED_MODEL, messages=[{"role": "user", "content": "sdk"}]
|
||||
)
|
||||
|
||||
_run(call)
|
||||
|
||||
|
||||
def _anthropic_messages(gateway: Gateway, key: str) -> None:
|
||||
with (
|
||||
httpx.Client(timeout=15, trust_env=False) as transport,
|
||||
anthropic.Anthropic(api_key=key, base_url=_origin(gateway), max_retries=0, http_client=transport) as client,
|
||||
):
|
||||
client.messages.create(model=SHARED_MODEL, max_tokens=16, messages=[{"role": "user", "content": "sdk"}])
|
||||
|
||||
|
||||
def _async_anthropic_messages(gateway: Gateway, key: str) -> None:
|
||||
async def call() -> object:
|
||||
async with (
|
||||
httpx.AsyncClient(timeout=15, trust_env=False) as transport,
|
||||
anthropic.AsyncAnthropic(
|
||||
api_key=key, base_url=_origin(gateway), max_retries=0, http_client=transport
|
||||
) as client,
|
||||
):
|
||||
return await client.messages.create(
|
||||
model=SHARED_MODEL, max_tokens=16, messages=[{"role": "user", "content": "sdk"}]
|
||||
)
|
||||
|
||||
_run(call)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("call", "error"),
|
||||
(
|
||||
pytest.param(_openai_chat, openai.BadRequestError, id="openai-sync-chat"),
|
||||
pytest.param(_async_openai_chat, openai.BadRequestError, id="openai-async-chat"),
|
||||
pytest.param(_openai_responses, openai.BadRequestError, id="openai-sync-responses"),
|
||||
pytest.param(_openai_embeddings, openai.BadRequestError, id="openai-sync-embeddings"),
|
||||
pytest.param(_anthropic_messages, anthropic.BadRequestError, id="anthropic-sync-messages"),
|
||||
pytest.param(_async_anthropic_messages, anthropic.BadRequestError, id="anthropic-async-messages"),
|
||||
),
|
||||
)
|
||||
def test_sdk_clients_get_a_bad_request_error_for_a_shared_raw_model_string(
|
||||
guarded: Gateway, peer: Wire, call: SdkCall, error: type[Exception]
|
||||
) -> None:
|
||||
with guarded.scenario() as scenario:
|
||||
peer.drain()
|
||||
|
||||
with pytest.raises(error, match="more than one model group"):
|
||||
call(guarded, scenario.key())
|
||||
assert peer.drain() == ()
|
||||
|
||||
|
||||
def test_a_model_group_name_still_routes_inside_its_own_group_when_the_check_is_on(
|
||||
guarded: Gateway, peer: Wire
|
||||
) -> None:
|
||||
with guarded.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
peer.drain()
|
||||
responses: Final = tuple(
|
||||
guarded.request("POST", *_chat("collision-chain-a", uuid.uuid4().hex), key=key) for _ in range(8)
|
||||
)
|
||||
served: Final = _served_by(peer.drain())
|
||||
|
||||
assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses]
|
||||
assert len(served) == 8
|
||||
assert set(served) <= CHAIN_A
|
||||
|
||||
|
||||
def test_a_raw_model_string_used_by_one_group_is_still_served_when_the_check_is_on(
|
||||
guarded: Gateway, peer: Wire
|
||||
) -> None:
|
||||
with guarded.scenario() as scenario:
|
||||
peer.drain()
|
||||
response: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=scenario.key())
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _served_by(peer.drain()) == ("collision-solo-1",)
|
||||
|
||||
|
||||
def test_without_the_setting_a_shared_raw_model_string_keeps_pooling_every_group(
|
||||
unguarded: Gateway, peer: Wire
|
||||
) -> None:
|
||||
with unguarded.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
peer.drain()
|
||||
responses: Final = tuple(
|
||||
unguarded.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) for _ in range(POOLED_CALLS)
|
||||
)
|
||||
served: Final = set(_served_by(peer.drain()))
|
||||
|
||||
assert [response.status_code for response in responses] == [200] * POOLED_CALLS, [r.text for r in responses]
|
||||
assert "collision-b-1" in served
|
||||
assert served & CHAIN_A
|
||||
assert served <= CHAIN_A | {"collision-b-1"}
|
||||
|
||||
|
||||
def _team_deployment(
|
||||
scenario: Scenario, wire: Wire, team_id: str, public_name: str, identity: str, model: str = TEAM_MODEL
|
||||
) -> None:
|
||||
created: Final = scenario.gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": public_name,
|
||||
"litellm_params": {
|
||||
"model": model,
|
||||
"api_base": f"{wire.url}/{identity}/v1",
|
||||
"api_key": "synthetic-key",
|
||||
},
|
||||
"model_info": {"team_id": team_id},
|
||||
},
|
||||
)
|
||||
scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"]))
|
||||
|
||||
|
||||
def test_the_deployments_of_one_team_model_count_as_one_model_group(guarded: Gateway, peer: Wire) -> None:
|
||||
with guarded.scenario() as scenario:
|
||||
team: Final = scenario.team()
|
||||
public_name: Final = f"collision-team-model-{uuid.uuid4().hex}"
|
||||
for identity in sorted(TEAM_DEPLOYMENTS):
|
||||
_team_deployment(scenario, peer, team, public_name, identity)
|
||||
key: Final = scenario.key(team_id=team)
|
||||
peer.drain()
|
||||
responses: Final = tuple(
|
||||
guarded.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(8)
|
||||
)
|
||||
served: Final = _served_by(peer.drain())
|
||||
|
||||
assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses]
|
||||
assert len(served) == 8
|
||||
assert set(served) <= TEAM_DEPLOYMENTS
|
||||
|
||||
|
||||
def test_another_teams_model_on_the_same_raw_model_string_does_not_refuse_this_team(
|
||||
guarded: Gateway, peer: Wire
|
||||
) -> None:
|
||||
with guarded.scenario() as scenario:
|
||||
team: Final = scenario.team()
|
||||
other_team: Final = scenario.team()
|
||||
public_name: Final = f"collision-team-model-{uuid.uuid4().hex}"
|
||||
for identity in sorted(TEAM_DEPLOYMENTS):
|
||||
_team_deployment(scenario, peer, team, public_name, identity)
|
||||
_team_deployment(scenario, peer, other_team, public_name, "collision-other-team-1")
|
||||
key: Final = scenario.key(team_id=team)
|
||||
peer.drain()
|
||||
responses: Final = tuple(
|
||||
guarded.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(8)
|
||||
)
|
||||
served: Final = _served_by(peer.drain())
|
||||
|
||||
assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses]
|
||||
assert len(served) == 8
|
||||
assert set(served) <= TEAM_DEPLOYMENTS
|
||||
|
||||
|
||||
def test_a_team_model_on_a_raw_model_string_a_proxy_group_also_uses_is_refused_for_that_team(
|
||||
guarded: Gateway, peer: Wire
|
||||
) -> None:
|
||||
with guarded.scenario() as scenario:
|
||||
team: Final = scenario.team()
|
||||
_team_deployment(
|
||||
scenario, peer, team, f"collision-team-model-{uuid.uuid4().hex}", "collision-team-solo-1", SOLO_MODEL
|
||||
)
|
||||
team_key: Final = scenario.key(team_id=team)
|
||||
other_key: Final = scenario.key()
|
||||
peer.drain()
|
||||
refused: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=team_key)
|
||||
served: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=other_key)
|
||||
received: Final = _served_by(peer.drain())
|
||||
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert SOLO_MODEL in refused.text
|
||||
assert served.status_code == 200, served.text
|
||||
assert received == ("collision-solo-1",)
|
||||
|
||||
|
||||
def test_a_team_model_created_before_public_names_counts_as_one_model_group(
|
||||
guarded_with_team_models_from_before_public_names: Gateway, peer: Wire
|
||||
) -> None:
|
||||
gateway: Final = guarded_with_team_models_from_before_public_names
|
||||
with gateway.scenario() as scenario:
|
||||
scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES)
|
||||
_team_deployment(scenario, peer, TEAM_BEFORE_PUBLIC_NAMES, "collision-team-model", "collision-team-3")
|
||||
key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES)
|
||||
peer.drain()
|
||||
responses: Final = tuple(
|
||||
gateway.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(12)
|
||||
)
|
||||
served: Final = _served_by(peer.drain())
|
||||
|
||||
assert [response.status_code for response in responses] == [200] * 12, [r.text for r in responses]
|
||||
assert len(served) == 12
|
||||
assert set(served) <= {*TEAM_DEPLOYMENTS, "collision-team-3"}
|
||||
|
||||
|
||||
def test_a_team_model_created_before_public_names_on_a_proxy_groups_raw_model_string_is_refused(
|
||||
guarded_with_team_models_from_before_public_names: Gateway, peer: Wire
|
||||
) -> None:
|
||||
gateway: Final = guarded_with_team_models_from_before_public_names
|
||||
with gateway.scenario() as scenario:
|
||||
scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES)
|
||||
team_key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES)
|
||||
other_key: Final = scenario.key()
|
||||
peer.drain()
|
||||
refused: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=team_key)
|
||||
served: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=other_key)
|
||||
received: Final = _served_by(peer.drain())
|
||||
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert SOLO_MODEL in refused.text
|
||||
assert served.status_code == 200, served.text
|
||||
assert received == ("collision-solo-1",)
|
||||
|
||||
|
||||
def _router_setting(gateway: Gateway, name: str) -> JsonValue:
|
||||
fields: Final = gateway.get("/router/settings")["fields"]
|
||||
assert isinstance(fields, list)
|
||||
return next(field["field_value"] for field in fields if isinstance(field, dict) and field["field_name"] == name)
|
||||
|
||||
|
||||
def test_the_admin_ui_config_update_turns_the_check_on_and_off_without_a_restart(toggled: Gateway, peer: Wire) -> None:
|
||||
with toggled.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
peer.drain()
|
||||
before: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key)
|
||||
toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": True}})
|
||||
enabled_value: Final = _router_setting(toggled, "enable_cross_model_group_collision_check")
|
||||
refused: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key)
|
||||
toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": False}})
|
||||
after: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key)
|
||||
served: Final = _served_by(peer.drain())
|
||||
|
||||
assert before.status_code == 200, before.text
|
||||
assert enabled_value is True
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert after.status_code == 200, after.text
|
||||
assert len(served) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "enabled"),
|
||||
(
|
||||
pytest.param("TRUE", True, id="upper-case-true"),
|
||||
pytest.param("yes", False, id="yes"),
|
||||
pytest.param(1, False, id="integer-one"),
|
||||
pytest.param("", False, id="empty-string"),
|
||||
),
|
||||
)
|
||||
def test_only_a_true_value_from_the_admin_ui_turns_the_check_on(
|
||||
toggled: Gateway, peer: Wire, value: JsonValue, enabled: bool
|
||||
) -> None:
|
||||
with toggled.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
scenario.cleanups.callback(
|
||||
toggled.post, "/config/update", {"router_settings": {"enable_cross_model_group_collision_check": False}}
|
||||
)
|
||||
toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": value}})
|
||||
peer.drain()
|
||||
response: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key)
|
||||
served: Final = _served_by(peer.drain())
|
||||
|
||||
assert response.status_code == (400 if enabled else 200), response.text
|
||||
assert len(served) == (0 if enabled else 1)
|
||||
|
||||
|
||||
def _burst_call(gateway: Gateway, key: str, index: int) -> tuple[bool, int | None, str]:
|
||||
raw: Final = index % 2 == 0
|
||||
build: Final = (_chat, _messages, _responses)[index // 2 % 3]
|
||||
marker: Final = f"collision-burst-{index}-{uuid.uuid4().hex}"
|
||||
path, body = build(SHARED_MODEL if raw else "collision-chain-b", marker)
|
||||
try:
|
||||
response: Final = gateway.request("POST", path, {**body, "stream": raw and index % 4 == 0}, key=key)
|
||||
except httpx.HTTPError:
|
||||
return raw, None, marker
|
||||
return raw, response.status_code, marker
|
||||
|
||||
|
||||
@pytest.mark.timeout(2 * graceful_stop_seconds() + 180)
|
||||
def test_the_refusal_holds_on_both_workers_through_a_worker_kill_mid_burst(peer: Wire, tmp_path: Path) -> None:
|
||||
path: Final = _config(peer, tmp_path, {"enable_cross_model_group_collision_check": True})
|
||||
with (
|
||||
gateway_from_environment() as base,
|
||||
owned_proxy_process(base, tmp_path, {}, config=path, workers=2) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
key: Final = scenario.key()
|
||||
workers: Final = eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30)
|
||||
peer.drain()
|
||||
with ThreadPoolExecutor(max_workers=BURST) as pool:
|
||||
futures: Final = [pool.submit(_burst_call, owned.gateway, key, index) for index in range(BURST)]
|
||||
with owned.replacing_worker(min(workers)) as kill:
|
||||
kill()
|
||||
outcomes: Final = [future.result() for future in futures]
|
||||
upstream_bodies: Final = tuple(request.body.decode() for request in peer.drain())
|
||||
probes: Final = tuple(
|
||||
owned.gateway.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) for _ in range(8)
|
||||
)
|
||||
control: Final = owned.gateway.request("POST", *_chat("collision-chain-b", uuid.uuid4().hex), key=key)
|
||||
|
||||
assert all(status in (400, None) for raw, status, _ in outcomes if raw), outcomes
|
||||
assert any(status == 400 for raw, status, _ in outcomes if raw), outcomes
|
||||
assert all(status in (200, None) for raw, status, _ in outcomes if not raw), outcomes
|
||||
assert not any(marker in body for raw, _, marker in outcomes if raw for body in upstream_bodies)
|
||||
assert [probe.status_code for probe in probes] == [400] * 8, [probe.text for probe in probes]
|
||||
assert control.status_code == 200, control.text
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import contextlib
|
||||
import logging
|
||||
from itertools import chain
|
||||
from typing import Dict, Final, List, Optional, Union
|
||||
|
|
@ -17,13 +18,21 @@ from litellm.router_utils.common_utils import (
|
|||
filter_web_search_deployments,
|
||||
format_fallback_outcome_message,
|
||||
provider_for_generic_call,
|
||||
refuse_litellm_model_spanning_model_groups,
|
||||
resolve_model_group_alias,
|
||||
resolve_served_model,
|
||||
truncate_fallback_error_detail,
|
||||
PROVIDER_SCOPED_CREDENTIAL_PARAMS,
|
||||
warn_on_provider_credential_mismatch,
|
||||
)
|
||||
from litellm.types.router import Deployment, FallbackSkipReason, LiteLLM_Params, ModelInfo, SkippedFallbackTarget
|
||||
from litellm.types.router import (
|
||||
Deployment,
|
||||
DeploymentTypedDict,
|
||||
FallbackSkipReason,
|
||||
LiteLLM_Params,
|
||||
ModelInfo,
|
||||
SkippedFallbackTarget,
|
||||
)
|
||||
from litellm.types.utils import ProviderSpecificModelInfo
|
||||
|
||||
|
||||
|
|
@ -957,3 +966,79 @@ def test_router_warns_once_and_serves_no_flag_for_a_deployment_synced_from_the_d
|
|||
assert len(capability_warnings) == 1, capability_warnings
|
||||
assert "db-model" in capability_warnings[0]
|
||||
assert "supports_max_reasoning_effort" not in router.get_model_list()[0]["litellm_params"]
|
||||
|
||||
|
||||
def _group_member(model_name: str, team: tuple[str, str | None] | None = None) -> DeploymentTypedDict:
|
||||
model_info: Final[dict[str, str]] = {}
|
||||
if team is not None:
|
||||
model_info["team_id"] = team[0]
|
||||
if team[1] is not None:
|
||||
model_info["team_public_model_name"] = team[1]
|
||||
return {"model_name": model_name, "litellm_params": {"model": "openai/shared-backend"}, "model_info": model_info}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployments", "refused"),
|
||||
(
|
||||
pytest.param((_group_member("chain-a"), _group_member("chain-b")), True, id="two-groups"),
|
||||
pytest.param((_group_member("chain-a"), _group_member("chain-a")), False, id="one-group"),
|
||||
pytest.param((_group_member("chain-a"),), False, id="one-deployment"),
|
||||
pytest.param((), False, id="no-deployments"),
|
||||
pytest.param(
|
||||
(
|
||||
_group_member("model_name_t1_a", ("t1", "gpt-team")),
|
||||
_group_member("model_name_t1_b", ("t1", "gpt-team")),
|
||||
),
|
||||
False,
|
||||
id="one-team-model",
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
_group_member("model_name_t1_a", ("t1", "gpt-team")),
|
||||
_group_member("model_name_t2_a", ("t2", "gpt-team")),
|
||||
),
|
||||
True,
|
||||
id="two-teams-same-public-name",
|
||||
),
|
||||
pytest.param(
|
||||
(_group_member("model_name_t1_a", ("t1", "gpt-team")), _group_member("gpt-team")),
|
||||
True,
|
||||
id="team-model-and-proxy-group",
|
||||
),
|
||||
pytest.param(
|
||||
(_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", None))),
|
||||
False,
|
||||
id="team-model-created-before-public-names",
|
||||
),
|
||||
pytest.param(
|
||||
(_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", "gpt-team"))),
|
||||
False,
|
||||
id="team-deployments-from-before-and-after-public-names",
|
||||
),
|
||||
pytest.param(
|
||||
(_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t2_a", ("t2", None))),
|
||||
True,
|
||||
id="two-teams-created-before-public-names",
|
||||
),
|
||||
pytest.param(
|
||||
(_group_member("model_name_t1_a", ("t1", None)), _group_member("chain-a")),
|
||||
True,
|
||||
id="team-model-created-before-public-names-and-proxy-group",
|
||||
),
|
||||
pytest.param(
|
||||
(_group_member("gpt-team", ("t1", None)), _group_member("chain-a")),
|
||||
True,
|
||||
id="team-deployment-named-like-a-proxy-group",
|
||||
),
|
||||
pytest.param(
|
||||
(_group_member("gpt-team", ("t1", None)), _group_member("gpt-team")),
|
||||
False,
|
||||
id="team-deployment-in-a-proxy-group",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_refuse_litellm_model_spanning_model_groups_decides_on_distinct_model_groups(
|
||||
deployments: tuple[DeploymentTypedDict, ...], refused: bool
|
||||
) -> None:
|
||||
with pytest.raises(litellm.BadRequestError) if refused else contextlib.nullcontext():
|
||||
refuse_litellm_model_spanning_model_groups("openai/shared-backend", deployments)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.types.router import RoutingStrategyName
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -425,3 +426,259 @@ def test_get_routable_deployments_returns_every_deployment_routing_can_pick(
|
|||
}
|
||||
|
||||
assert routable_ids == expected_ids
|
||||
|
||||
|
||||
_SHARED_LITELLM_MODEL: Final = "openai/shared-backend"
|
||||
_COLLIDING_MODEL_LIST: Final = [
|
||||
{
|
||||
"model_name": "chain-a",
|
||||
"litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"},
|
||||
"model_info": {"id": "chain-a-1"},
|
||||
},
|
||||
{
|
||||
"model_name": "chain-a",
|
||||
"litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"},
|
||||
"model_info": {"id": "chain-a-2"},
|
||||
},
|
||||
{
|
||||
"model_name": "chain-b",
|
||||
"litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"},
|
||||
"model_info": {"id": "chain-b-1"},
|
||||
},
|
||||
{
|
||||
"model_name": "solo",
|
||||
"litellm_params": {"model": "openai/solo-backend", "api_key": "test-key"},
|
||||
"model_info": {"id": "solo-1"},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _pick_sync(router: Router, model: str, specific_deployment: bool) -> str:
|
||||
return router.get_available_deployment(model=model, specific_deployment=specific_deployment)["model_info"]["id"] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict
|
||||
|
||||
|
||||
def _pick_async(router: Router, model: str, specific_deployment: bool) -> str:
|
||||
async def pick() -> str:
|
||||
deployment: Final = await router.async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict
|
||||
model=model, request_kwargs={}, specific_deployment=specific_deployment
|
||||
)
|
||||
return deployment["model_info"]["id"] # pyright: ignore[reportUnknownVariableType] # the router returns an untyped deployment dict
|
||||
|
||||
return asyncio.run(pick())
|
||||
|
||||
|
||||
_PICKS: Final = (
|
||||
pytest.param(_pick_sync, False, id="sync"),
|
||||
pytest.param(_pick_sync, True, id="sync-specific-deployment"),
|
||||
pytest.param(_pick_async, False, id="async"),
|
||||
pytest.param(_pick_async, True, id="async-specific-deployment"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS)
|
||||
def test_collision_check_refuses_a_litellm_model_shared_by_two_model_groups(
|
||||
pick: Callable[[Router, str, bool], str], specific_deployment: bool
|
||||
) -> None:
|
||||
router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
|
||||
pick(router, _SHARED_LITELLM_MODEL, specific_deployment)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS)
|
||||
def test_collision_check_off_keeps_pooling_a_shared_litellm_model_across_groups(
|
||||
pick: Callable[[Router, str, bool], str], specific_deployment: bool
|
||||
) -> None:
|
||||
router: Final = Router(model_list=_COLLIDING_MODEL_LIST)
|
||||
|
||||
picked: Final = {pick(router, _SHARED_LITELLM_MODEL, specific_deployment) for _ in range(60)}
|
||||
|
||||
assert picked == {"chain-a-1", "chain-a-2", "chain-b-1"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS)
|
||||
def test_collision_check_still_serves_a_litellm_model_used_by_a_single_group(
|
||||
pick: Callable[[Router, str, bool], str], specific_deployment: bool
|
||||
) -> None:
|
||||
router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True)
|
||||
|
||||
assert pick(router, "openai/solo-backend", specific_deployment) == "solo-1"
|
||||
|
||||
|
||||
def test_collision_check_leaves_model_group_names_routing_inside_their_group() -> None:
|
||||
router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True)
|
||||
|
||||
picked: Final = {_pick_sync(router, "chain-a", False) for _ in range(40)}
|
||||
|
||||
assert picked == {"chain-a-1", "chain-a-2"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enabled", (True, False))
|
||||
def test_collision_check_applies_to_an_alias_that_points_at_a_shared_litellm_model(enabled: bool) -> None:
|
||||
router: Final = Router(
|
||||
model_list=_COLLIDING_MODEL_LIST,
|
||||
model_group_alias={"alias": _SHARED_LITELLM_MODEL},
|
||||
enable_cross_model_group_collision_check=enabled,
|
||||
)
|
||||
|
||||
if enabled:
|
||||
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
|
||||
_pick_sync(router, "alias", False)
|
||||
else:
|
||||
assert {_pick_sync(router, "alias", False) for _ in range(60)} == {"chain-a-1", "chain-a-2", "chain-b-1"}
|
||||
|
||||
|
||||
def _team_deployment(deployment_id: str, team_id: str, public_name: str | None = "team-model") -> dict[str, object]:
|
||||
model_info: Final[dict[str, str]] = {"id": deployment_id, "team_id": team_id}
|
||||
if public_name is not None:
|
||||
model_info["team_public_model_name"] = public_name
|
||||
return {
|
||||
"model_name": f"model_name_{team_id}_{deployment_id}",
|
||||
"litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"},
|
||||
"model_info": model_info,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("specific_deployment", (True, False))
|
||||
def test_collision_check_treats_the_deployments_of_one_team_model_as_one_model_group(
|
||||
specific_deployment: bool,
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[_team_deployment("team-1-a", "team-1"), _team_deployment("team-1-b", "team-1")],
|
||||
enable_cross_model_group_collision_check=True,
|
||||
)
|
||||
|
||||
picked: Final = {_pick_sync(router, _SHARED_LITELLM_MODEL, specific_deployment) for _ in range(40)}
|
||||
|
||||
assert picked == {"team-1-a", "team-1-b"}
|
||||
|
||||
|
||||
def _pick_for_team(router: Router, team_id: str, specific_deployment: bool) -> str:
|
||||
async def pick() -> str:
|
||||
deployment: Final = await router.async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict
|
||||
model=_SHARED_LITELLM_MODEL,
|
||||
request_kwargs={"metadata": {"user_api_key_team_id": team_id}},
|
||||
specific_deployment=specific_deployment,
|
||||
)
|
||||
return deployment["model_info"]["id"] # pyright: ignore[reportUnknownVariableType] # the router returns an untyped deployment dict
|
||||
|
||||
return asyncio.run(pick())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("specific_deployment", (True, False))
|
||||
@pytest.mark.parametrize(
|
||||
"public_names",
|
||||
(
|
||||
pytest.param((None, None), id="both-created-before-public-names"),
|
||||
pytest.param((None, "team-model"), id="one-created-before-public-names"),
|
||||
),
|
||||
)
|
||||
def test_collision_check_serves_a_team_model_created_before_public_names(
|
||||
public_names: tuple[str | None, str | None], specific_deployment: bool
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
_team_deployment("team-1-a", "team-1", public_names[0]),
|
||||
_team_deployment("team-1-b", "team-1", public_names[1]),
|
||||
],
|
||||
enable_cross_model_group_collision_check=True,
|
||||
)
|
||||
|
||||
picked: Final = {_pick_for_team(router, "team-1", specific_deployment) for _ in range(40)}
|
||||
|
||||
assert picked == {"team-1-a", "team-1-b"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("specific_deployment", (True, False))
|
||||
def test_collision_check_serves_each_team_its_own_model_when_another_team_shares_the_litellm_model(
|
||||
specific_deployment: bool,
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[_team_deployment("team-1-a", "team-1"), _team_deployment("team-2-a", "team-2")],
|
||||
enable_cross_model_group_collision_check=True,
|
||||
)
|
||||
|
||||
assert _pick_for_team(router, "team-1", specific_deployment) == "team-1-a"
|
||||
assert _pick_for_team(router, "team-2", specific_deployment) == "team-2-a"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("specific_deployment", (True, False))
|
||||
def test_collision_check_refuses_a_team_whose_model_shares_the_litellm_model_with_a_proxy_group(
|
||||
specific_deployment: bool,
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[_team_deployment("team-1-a", "team-1"), _COLLIDING_MODEL_LIST[2]],
|
||||
enable_cross_model_group_collision_check=True,
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
|
||||
_pick_for_team(router, "team-1", specific_deployment)
|
||||
assert _pick_for_team(router, "team-2", specific_deployment) == "chain-b-1"
|
||||
|
||||
|
||||
def _served_model_list() -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
**deployment,
|
||||
"litellm_params": {
|
||||
**deployment["litellm_params"],
|
||||
"mock_response": f"served by {deployment['model_info']['id']}",
|
||||
},
|
||||
}
|
||||
for deployment in _COLLIDING_MODEL_LIST
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"fallbacks",
|
||||
(
|
||||
pytest.param([{"*": ["solo"]}], id="wildcard-fallback"),
|
||||
pytest.param([{"alias": ["solo"]}, {_SHARED_LITELLM_MODEL: ["solo"]}], id="keyed-fallback"),
|
||||
),
|
||||
)
|
||||
@pytest.mark.parametrize("model", ("alias", _SHARED_LITELLM_MODEL))
|
||||
def test_collision_check_refusal_is_not_rescued_by_a_fallback(
|
||||
fallbacks: list[dict[str, list[str]]], model: str
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=_served_model_list(),
|
||||
model_group_alias={"alias": _SHARED_LITELLM_MODEL},
|
||||
fallbacks=fallbacks,
|
||||
enable_cross_model_group_collision_check=True,
|
||||
)
|
||||
messages: Final = [{"role": "user", "content": "hi"}]
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
|
||||
router.completion(model=model, messages=messages)
|
||||
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
|
||||
asyncio.run(router.acompletion(model=model, messages=messages))
|
||||
assert router.completion(model="chain-b", messages=messages).choices[0].message.content == "served by chain-b-1" # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # completion returns an untyped response union
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("setting", "enabled"),
|
||||
((True, True), ("true", True), (False, False), ("false", False)),
|
||||
)
|
||||
def test_collision_check_can_be_toggled_through_update_settings(setting: bool | str, enabled: bool) -> None:
|
||||
router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=not enabled)
|
||||
|
||||
router.update_settings(enable_cross_model_group_collision_check=setting) # pyright: ignore[reportUnknownMemberType] # update_settings takes untyped **kwargs
|
||||
|
||||
assert len(router.get_routable_deployments(_SHARED_LITELLM_MODEL)) == (0 if enabled else 3)
|
||||
assert router.get_settings()["enable_cross_model_group_collision_check"] is enabled
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"routing_strategy",
|
||||
("simple-shuffle", "least-busy", "usage-based-routing-v2", "latency-based-routing", "cost-based-routing"),
|
||||
)
|
||||
def test_collision_check_refuses_under_every_routing_strategy(routing_strategy: RoutingStrategyName) -> None:
|
||||
router: Final = Router(
|
||||
model_list=_COLLIDING_MODEL_LIST,
|
||||
routing_strategy=routing_strategy,
|
||||
enable_cross_model_group_collision_check=True,
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
|
||||
_pick_async(router, _SHARED_LITELLM_MODEL, False)
|
||||
assert _pick_async(router, "chain-a", False) in {"chain-a-1", "chain-a-2"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue