mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router): keep the endpoint gate on model_info patches that omit it and on same-group stream retries
A partial PATCH of model_info reset a stored enforce_supported_endpoints true back to false, because the field defaulted to False and the update merges model_dump(exclude_none=True). The field now defaults to None so an omitted flag keeps the stored value The same-group retry of a stream dropped before content called the provider without the endpoint selection context, so it could land on an enforced embedding-only deployment of the same group. It now runs inside endpoint_selection like every other router attempt
This commit is contained in:
parent
9d5ab2729b
commit
3a33a45e64
8 changed files with 115 additions and 14 deletions
|
|
@ -6767,6 +6767,7 @@ class Router:
|
|||
"""
|
||||
model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group
|
||||
retry_kwargs: Final = mid_stream_retry_kwargs(initial_kwargs)
|
||||
handler: Final = retry_kwargs.get("original_generic_function")
|
||||
healthy_deployments, all_deployments = await self._async_get_healthy_deployments(
|
||||
model=model_group,
|
||||
parent_otel_span=get_parent_otel_span_from_kwargs(retry_kwargs),
|
||||
|
|
@ -6797,7 +6798,10 @@ class Router:
|
|||
)
|
||||
carry_superseded_attempt_usage(retry_kwargs)
|
||||
try:
|
||||
response = await attempt_function(**retry_kwargs)
|
||||
with endpoint_selection(
|
||||
getattr(handler, "__name__", None), getattr(handler, "__module__", None), model=model_group
|
||||
):
|
||||
response = await attempt_function(**retry_kwargs)
|
||||
except Exception as retry_error: # noqa: BLE001 # the fallback chain judges the failed retry
|
||||
wrapped = _stream_fallback_error_for_raised(retry_error, model_group, False, e.llm_provider)
|
||||
if wrapped is None:
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ _REQUEST_SELECTION: Final[ContextVar[tuple[str | None, str | None]]] = ContextVa
|
|||
class _EndpointDeclaration(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
enforce_supported_endpoints: bool = False
|
||||
enforce_supported_endpoints: bool | None = None
|
||||
supported_endpoints: tuple[str, ...] | None = None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -265,7 +265,7 @@ def _whole_float_as_int(value: object) -> object:
|
|||
|
||||
|
||||
class ModelInfo(MirroredPricingParams):
|
||||
enforce_supported_endpoints: bool = False
|
||||
enforce_supported_endpoints: bool | None = None
|
||||
id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance
|
||||
db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config.
|
||||
updated_at: datetime.datetime | None = None
|
||||
|
|
|
|||
|
|
@ -3601,6 +3601,33 @@ class TestDeleteTeamModels:
|
|||
assert any(e[0] == "delete_many" for e in prisma.events)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_info_patch", "expected_gate"),
|
||||
[
|
||||
pytest.param({"tpm": 5}, True, id="unrelated-field"),
|
||||
pytest.param({"supported_endpoints": ["/v1/embeddings", "/v1/rerank"]}, True, id="endpoint-list-only"),
|
||||
pytest.param({"enforce_supported_endpoints": False}, False, id="explicit-false"),
|
||||
],
|
||||
)
|
||||
def test_model_info_patch_changes_the_endpoint_gate_only_when_it_sends_the_flag(
|
||||
model_info_patch: dict[str, object], expected_gate: bool
|
||||
):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
|
||||
result: Final = update_db_model(
|
||||
db_model=Deployment(
|
||||
model_name="embedding-only",
|
||||
litellm_params=LiteLLM_Params(model="openai/text-embedding-3-small"),
|
||||
model_info=ModelInfo(
|
||||
id="embedding-only", enforce_supported_endpoints=True, supported_endpoints=["/v1/embeddings"]
|
||||
),
|
||||
),
|
||||
updated_patch=updateDeployment.model_validate({"model_info": model_info_patch}),
|
||||
)
|
||||
|
||||
assert json.loads(result["model_info"]).get("enforce_supported_endpoints") is expected_gate
|
||||
|
||||
|
||||
def _build_db_model_for_blocked_test():
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
|
|||
|
|
@ -716,9 +716,6 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
assert returned_data["metadata"]["tags"] == ["existing-tag"]
|
||||
tag_budget_check.assert_not_awaited()
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_processing_pre_call_logic_skips_guardrail_tag_budget_check_for_zero_cost_model(
|
||||
self, monkeypatch
|
||||
|
|
@ -7547,6 +7544,56 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
|
|||
assert processor.data["model"] == fallback_model
|
||||
assert call_count == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_rate_limit_fallback_skips_an_aliased_embedding_only_model(self):
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "embed-only",
|
||||
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-synthetic"},
|
||||
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
|
||||
},
|
||||
{"model_name": "chat-model", "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-synthetic"}},
|
||||
],
|
||||
fallbacks=[{"rate-limited": ["embed-alias", "chat-model"]}],
|
||||
)
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(
|
||||
data={
|
||||
"model": "rate-limited",
|
||||
"router_settings_override": {"model_group_alias": {"embed-alias": "embed-only"}},
|
||||
}
|
||||
)
|
||||
attempted: Final[list[object]] = []
|
||||
|
||||
async def pre_call_logic(**_: object):
|
||||
attempted.append(processor.data.get("model"))
|
||||
if processor.data.get("model") == "rate-limited":
|
||||
raise ProxyRateLimitError(detail="key rpm limit exceeded", headers={"retry-after": "30"})
|
||||
return processor.data, MagicMock()
|
||||
|
||||
with patch.object(processor, "common_processing_pre_call_logic", side_effect=pre_call_logic):
|
||||
await processor._pre_call_with_fallbacks(
|
||||
request=MagicMock(),
|
||||
general_settings={},
|
||||
proxy_logging_obj=MagicMock(),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(),
|
||||
version=None,
|
||||
proxy_config=MagicMock(),
|
||||
user_model=None,
|
||||
user_temperature=None,
|
||||
user_request_timeout=None,
|
||||
user_max_tokens=None,
|
||||
user_api_base=None,
|
||||
model="rate-limited",
|
||||
route_type="acompletion",
|
||||
llm_router=router,
|
||||
)
|
||||
|
||||
assert attempted == ["rate-limited", "chat-model"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_when_no_fallbacks_configured(self):
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
|
@ -7629,7 +7676,6 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
|
|||
|
||||
assert processor.data["model"] == "gpt-4"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_uses_key_level_router_settings(self):
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
|
|
|||
|
|
@ -2240,8 +2240,6 @@ async def test_route_request_router_settings_override_skips_null_fields():
|
|||
assert "model_group_retry_policy" not in call_kwargs
|
||||
|
||||
|
||||
|
||||
|
||||
def _router_with_defaults(default_litellm_params: dict[str, object]):
|
||||
import litellm
|
||||
|
||||
|
|
|
|||
|
|
@ -18242,6 +18242,35 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi
|
|||
assert [(attempted, budget) for _, attempted, budget in provider.calls] == [(0, 2), (1, 2)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_same_group_retry_skips_an_embedding_only_deployment():
|
||||
"""The retry of a /v1/messages stream dropped before content must not land on an enforced embedding-only
|
||||
deployment of the same group, even when that deployment is the group's preferred order."""
|
||||
router = Router(
|
||||
model_list=[
|
||||
{"model_name": "glm", "litellm_params": {"model": "anthropic/glm-chat", "api_key": "sk-test", "order": 2}},
|
||||
{
|
||||
"model_name": "glm",
|
||||
"litellm_params": {"model": "anthropic/glm-embed", "api_key": "sk-test", "order": 1},
|
||||
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
|
||||
},
|
||||
],
|
||||
num_retries=1,
|
||||
)
|
||||
provider: Final = _AnthropicMessagesScriptedProvider(
|
||||
_anthropic_messages_dropped_before_content, _anthropic_messages_retried_stream
|
||||
)
|
||||
|
||||
async def anthropic_messages(**kwargs):
|
||||
return await provider(**kwargs)
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, anthropic_messages)
|
||||
body: Final = [chunk async for chunk in stream]
|
||||
|
||||
assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("pong")]
|
||||
assert [model for model, _, _ in provider.calls] == ["anthropic/glm-chat", "anthropic/glm-chat"]
|
||||
|
||||
|
||||
class _AnthropicMessagesRetryStashRecorder:
|
||||
def __init__(self) -> None:
|
||||
self.observed: tuple[Usage | None, ...] = ()
|
||||
|
|
|
|||
7
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
7
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -50364,11 +50364,8 @@ export interface components {
|
|||
discoverable?: boolean | null;
|
||||
/** Enable Tag Filtering */
|
||||
enable_tag_filtering?: boolean | null;
|
||||
/**
|
||||
* Enforce Supported Endpoints
|
||||
* @default false
|
||||
*/
|
||||
enforce_supported_endpoints: boolean;
|
||||
/** Enforce Supported Endpoints */
|
||||
enforce_supported_endpoints?: boolean | null;
|
||||
/** Explicit Cache Read Input Token Cost */
|
||||
explicit_cache_read_input_token_cost?: number | null;
|
||||
/** Id */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue