fix(router): keep the endpoint gate on model_info patches that omit it and on same-group stream retries

A partial PATCH of model_info reset a stored enforce_supported_endpoints true back to false, because the
field defaulted to False and the update merges model_dump(exclude_none=True). The field now defaults to None
so an omitted flag keeps the stored value

The same-group retry of a stream dropped before content called the provider without the endpoint selection
context, so it could land on an enforced embedding-only deployment of the same group. It now runs inside
endpoint_selection like every other router attempt
This commit is contained in:
mateo-berri 2026-10-28 18:57:54 -07:00
parent 9d5ab2729b
commit 3a33a45e64
8 changed files with 115 additions and 14 deletions

View file

@ -6767,6 +6767,7 @@ class Router:
"""
model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group
retry_kwargs: Final = mid_stream_retry_kwargs(initial_kwargs)
handler: Final = retry_kwargs.get("original_generic_function")
healthy_deployments, all_deployments = await self._async_get_healthy_deployments(
model=model_group,
parent_otel_span=get_parent_otel_span_from_kwargs(retry_kwargs),
@ -6797,7 +6798,10 @@ class Router:
)
carry_superseded_attempt_usage(retry_kwargs)
try:
response = await attempt_function(**retry_kwargs)
with endpoint_selection(
getattr(handler, "__name__", None), getattr(handler, "__module__", None), model=model_group
):
response = await attempt_function(**retry_kwargs)
except Exception as retry_error: # noqa: BLE001 # the fallback chain judges the failed retry
wrapped = _stream_fallback_error_for_raised(retry_error, model_group, False, e.llm_provider)
if wrapped is None:

View file

@ -44,7 +44,7 @@ _REQUEST_SELECTION: Final[ContextVar[tuple[str | None, str | None]]] = ContextVa
class _EndpointDeclaration(BaseModel):
model_config = ConfigDict(frozen=True)
enforce_supported_endpoints: bool = False
enforce_supported_endpoints: bool | None = None
supported_endpoints: tuple[str, ...] | None = None

View file

@ -265,7 +265,7 @@ def _whole_float_as_int(value: object) -> object:
class ModelInfo(MirroredPricingParams):
enforce_supported_endpoints: bool = False
enforce_supported_endpoints: bool | None = None
id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance
db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config.
updated_at: datetime.datetime | None = None

View file

@ -3601,6 +3601,33 @@ class TestDeleteTeamModels:
assert any(e[0] == "delete_many" for e in prisma.events)
@pytest.mark.parametrize(
("model_info_patch", "expected_gate"),
[
pytest.param({"tpm": 5}, True, id="unrelated-field"),
pytest.param({"supported_endpoints": ["/v1/embeddings", "/v1/rerank"]}, True, id="endpoint-list-only"),
pytest.param({"enforce_supported_endpoints": False}, False, id="explicit-false"),
],
)
def test_model_info_patch_changes_the_endpoint_gate_only_when_it_sends_the_flag(
model_info_patch: dict[str, object], expected_gate: bool
):
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
result: Final = update_db_model(
db_model=Deployment(
model_name="embedding-only",
litellm_params=LiteLLM_Params(model="openai/text-embedding-3-small"),
model_info=ModelInfo(
id="embedding-only", enforce_supported_endpoints=True, supported_endpoints=["/v1/embeddings"]
),
),
updated_patch=updateDeployment.model_validate({"model_info": model_info_patch}),
)
assert json.loads(result["model_info"]).get("enforce_supported_endpoints") is expected_gate
def _build_db_model_for_blocked_test():
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo

View file

@ -716,9 +716,6 @@ class TestProxyBaseLLMRequestProcessing:
assert returned_data["metadata"]["tags"] == ["existing-tag"]
tag_budget_check.assert_not_awaited()
@pytest.mark.asyncio
async def test_common_processing_pre_call_logic_skips_guardrail_tag_budget_check_for_zero_cost_model(
self, monkeypatch
@ -7547,6 +7544,56 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
assert processor.data["model"] == fallback_model
assert call_count == 2
@pytest.mark.asyncio
async def test_local_rate_limit_fallback_skips_an_aliased_embedding_only_model(self):
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
router: Final = Router(
model_list=[
{
"model_name": "embed-only",
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-synthetic"},
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
},
{"model_name": "chat-model", "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-synthetic"}},
],
fallbacks=[{"rate-limited": ["embed-alias", "chat-model"]}],
)
processor: Final = ProxyBaseLLMRequestProcessing(
data={
"model": "rate-limited",
"router_settings_override": {"model_group_alias": {"embed-alias": "embed-only"}},
}
)
attempted: Final[list[object]] = []
async def pre_call_logic(**_: object):
attempted.append(processor.data.get("model"))
if processor.data.get("model") == "rate-limited":
raise ProxyRateLimitError(detail="key rpm limit exceeded", headers={"retry-after": "30"})
return processor.data, MagicMock()
with patch.object(processor, "common_processing_pre_call_logic", side_effect=pre_call_logic):
await processor._pre_call_with_fallbacks(
request=MagicMock(),
general_settings={},
proxy_logging_obj=MagicMock(),
user_api_key_dict=ProxyUserAPIKeyAuth(),
version=None,
proxy_config=MagicMock(),
user_model=None,
user_temperature=None,
user_request_timeout=None,
user_max_tokens=None,
user_api_base=None,
model="rate-limited",
route_type="acompletion",
llm_router=router,
)
assert attempted == ["rate-limited", "chat-model"]
@pytest.mark.asyncio
async def test_raises_when_no_fallbacks_configured(self):
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
@ -7629,7 +7676,6 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
assert processor.data["model"] == "gpt-4"
@pytest.mark.asyncio
async def test_fallback_uses_key_level_router_settings(self):
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError

View file

@ -2240,8 +2240,6 @@ async def test_route_request_router_settings_override_skips_null_fields():
assert "model_group_retry_policy" not in call_kwargs
def _router_with_defaults(default_litellm_params: dict[str, object]):
import litellm

View file

@ -18242,6 +18242,35 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi
assert [(attempted, budget) for _, attempted, budget in provider.calls] == [(0, 2), (1, 2)]
@pytest.mark.asyncio
async def test_anthropic_messages_same_group_retry_skips_an_embedding_only_deployment():
"""The retry of a /v1/messages stream dropped before content must not land on an enforced embedding-only
deployment of the same group, even when that deployment is the group's preferred order."""
router = Router(
model_list=[
{"model_name": "glm", "litellm_params": {"model": "anthropic/glm-chat", "api_key": "sk-test", "order": 2}},
{
"model_name": "glm",
"litellm_params": {"model": "anthropic/glm-embed", "api_key": "sk-test", "order": 1},
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
},
],
num_retries=1,
)
provider: Final = _AnthropicMessagesScriptedProvider(
_anthropic_messages_dropped_before_content, _anthropic_messages_retried_stream
)
async def anthropic_messages(**kwargs):
return await provider(**kwargs)
stream: Final = await _anthropic_messages_stream_through_router(router, anthropic_messages)
body: Final = [chunk async for chunk in stream]
assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("pong")]
assert [model for model, _, _ in provider.calls] == ["anthropic/glm-chat", "anthropic/glm-chat"]
class _AnthropicMessagesRetryStashRecorder:
def __init__(self) -> None:
self.observed: tuple[Usage | None, ...] = ()

View file

@ -50364,11 +50364,8 @@ export interface components {
discoverable?: boolean | null;
/** Enable Tag Filtering */
enable_tag_filtering?: boolean | null;
/**
* Enforce Supported Endpoints
* @default false
*/
enforce_supported_endpoints: boolean;
/** Enforce Supported Endpoints */
enforce_supported_endpoints?: boolean | null;
/** Explicit Cache Read Input Token Cost */
explicit_cache_read_input_token_cost?: number | null;
/** Id */