diff --git a/litellm/router.py b/litellm/router.py index efe2bccfb9a..9b7ec685ed2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6767,6 +6767,7 @@ class Router: """ model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group retry_kwargs: Final = mid_stream_retry_kwargs(initial_kwargs) + handler: Final = retry_kwargs.get("original_generic_function") healthy_deployments, all_deployments = await self._async_get_healthy_deployments( model=model_group, parent_otel_span=get_parent_otel_span_from_kwargs(retry_kwargs), @@ -6797,7 +6798,10 @@ class Router: ) carry_superseded_attempt_usage(retry_kwargs) try: - response = await attempt_function(**retry_kwargs) + with endpoint_selection( + getattr(handler, "__name__", None), getattr(handler, "__module__", None), model=model_group + ): + response = await attempt_function(**retry_kwargs) except Exception as retry_error: # noqa: BLE001 # the fallback chain judges the failed retry wrapped = _stream_fallback_error_for_raised(retry_error, model_group, False, e.llm_provider) if wrapped is None: diff --git a/litellm/router_utils/endpoint_eligibility.py b/litellm/router_utils/endpoint_eligibility.py index 112ccaf535a..942d2d48572 100644 --- a/litellm/router_utils/endpoint_eligibility.py +++ b/litellm/router_utils/endpoint_eligibility.py @@ -44,7 +44,7 @@ _REQUEST_SELECTION: Final[ContextVar[tuple[str | None, str | None]]] = ContextVa class _EndpointDeclaration(BaseModel): model_config = ConfigDict(frozen=True) - enforce_supported_endpoints: bool = False + enforce_supported_endpoints: bool | None = None supported_endpoints: tuple[str, ...] | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index aa571f8d270..5fc2d9e1804 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -265,7 +265,7 @@ def _whole_float_as_int(value: object) -> object: class ModelInfo(MirroredPricingParams): - enforce_supported_endpoints: bool = False + enforce_supported_endpoints: bool | None = None id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config. updated_at: datetime.datetime | None = None diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index c8f601de280..6e5310d8a6c 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -3601,6 +3601,33 @@ class TestDeleteTeamModels: assert any(e[0] == "delete_many" for e in prisma.events) +@pytest.mark.parametrize( + ("model_info_patch", "expected_gate"), + [ + pytest.param({"tpm": 5}, True, id="unrelated-field"), + pytest.param({"supported_endpoints": ["/v1/embeddings", "/v1/rerank"]}, True, id="endpoint-list-only"), + pytest.param({"enforce_supported_endpoints": False}, False, id="explicit-false"), + ], +) +def test_model_info_patch_changes_the_endpoint_gate_only_when_it_sends_the_flag( + model_info_patch: dict[str, object], expected_gate: bool +): + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + + result: Final = update_db_model( + db_model=Deployment( + model_name="embedding-only", + litellm_params=LiteLLM_Params(model="openai/text-embedding-3-small"), + model_info=ModelInfo( + id="embedding-only", enforce_supported_endpoints=True, supported_endpoints=["/v1/embeddings"] + ), + ), + updated_patch=updateDeployment.model_validate({"model_info": model_info_patch}), + ) + + assert json.loads(result["model_info"]).get("enforce_supported_endpoints") is expected_gate + + def _build_db_model_for_blocked_test(): from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index bee4fd0f994..8b35d0e665e 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -716,9 +716,6 @@ class TestProxyBaseLLMRequestProcessing: assert returned_data["metadata"]["tags"] == ["existing-tag"] tag_budget_check.assert_not_awaited() - - - @pytest.mark.asyncio async def test_common_processing_pre_call_logic_skips_guardrail_tag_budget_check_for_zero_cost_model( self, monkeypatch @@ -7547,6 +7544,56 @@ class TestPreCallWithFallbacksOnLocalRateLimit: assert processor.data["model"] == fallback_model assert call_count == 2 + @pytest.mark.asyncio + async def test_local_rate_limit_fallback_skips_an_aliased_embedding_only_model(self): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + router: Final = Router( + model_list=[ + { + "model_name": "embed-only", + "litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-synthetic"}, + "model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]}, + }, + {"model_name": "chat-model", "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-synthetic"}}, + ], + fallbacks=[{"rate-limited": ["embed-alias", "chat-model"]}], + ) + processor: Final = ProxyBaseLLMRequestProcessing( + data={ + "model": "rate-limited", + "router_settings_override": {"model_group_alias": {"embed-alias": "embed-only"}}, + } + ) + attempted: Final[list[object]] = [] + + async def pre_call_logic(**_: object): + attempted.append(processor.data.get("model")) + if processor.data.get("model") == "rate-limited": + raise ProxyRateLimitError(detail="key rpm limit exceeded", headers={"retry-after": "30"}) + return processor.data, MagicMock() + + with patch.object(processor, "common_processing_pre_call_logic", side_effect=pre_call_logic): + await processor._pre_call_with_fallbacks( + request=MagicMock(), + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=ProxyUserAPIKeyAuth(), + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model="rate-limited", + route_type="acompletion", + llm_router=router, + ) + + assert attempted == ["rate-limited", "chat-model"] + @pytest.mark.asyncio async def test_raises_when_no_fallbacks_configured(self): from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError @@ -7629,7 +7676,6 @@ class TestPreCallWithFallbacksOnLocalRateLimit: assert processor.data["model"] == "gpt-4" - @pytest.mark.asyncio async def test_fallback_uses_key_level_router_settings(self): from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index a1f5aaf46e4..6044475652c 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -2240,8 +2240,6 @@ async def test_route_request_router_settings_override_skips_null_fields(): assert "model_group_retry_policy" not in call_kwargs - - def _router_with_defaults(default_litellm_params: dict[str, object]): import litellm diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 79f8e54fd76..8c61778206a 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -18242,6 +18242,35 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi assert [(attempted, budget) for _, attempted, budget in provider.calls] == [(0, 2), (1, 2)] +@pytest.mark.asyncio +async def test_anthropic_messages_same_group_retry_skips_an_embedding_only_deployment(): + """The retry of a /v1/messages stream dropped before content must not land on an enforced embedding-only + deployment of the same group, even when that deployment is the group's preferred order.""" + router = Router( + model_list=[ + {"model_name": "glm", "litellm_params": {"model": "anthropic/glm-chat", "api_key": "sk-test", "order": 2}}, + { + "model_name": "glm", + "litellm_params": {"model": "anthropic/glm-embed", "api_key": "sk-test", "order": 1}, + "model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]}, + }, + ], + num_retries=1, + ) + provider: Final = _AnthropicMessagesScriptedProvider( + _anthropic_messages_dropped_before_content, _anthropic_messages_retried_stream + ) + + async def anthropic_messages(**kwargs): + return await provider(**kwargs) + + stream: Final = await _anthropic_messages_stream_through_router(router, anthropic_messages) + body: Final = [chunk async for chunk in stream] + + assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("pong")] + assert [model for model, _, _ in provider.calls] == ["anthropic/glm-chat", "anthropic/glm-chat"] + + class _AnthropicMessagesRetryStashRecorder: def __init__(self) -> None: self.observed: tuple[Usage | None, ...] = () diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b61a1c81666..2eebb57b252 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -50364,11 +50364,8 @@ export interface components { discoverable?: boolean | null; /** Enable Tag Filtering */ enable_tag_filtering?: boolean | null; - /** - * Enforce Supported Endpoints - * @default false - */ - enforce_supported_endpoints: boolean; + /** Enforce Supported Endpoints */ + enforce_supported_endpoints?: boolean | null; /** Explicit Cache Read Input Token Cost */ explicit_cache_read_input_token_cost?: number | null; /** Id */