diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9022bd1392b..dc6a6f45e72 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -872,6 +872,17 @@ _DECODED_JSON: Final = TypeAdapter(object) _ROUTER_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +def _fallback_probe_data( + pristine: object, fallback_model: str, key_aliases: object, group_alias: object +) -> Mapping[str, object]: + return { + **_REQUEST_FIELDS.validate_python(pristine), + "model": resolve_served_model( + fallback_model, _DECODED_JSON.validate_python(litellm.model_alias_map), key_aliases, group_alias + ), + } + + def _request_model(data: Mapping[str, object]) -> str | list[str] | None: try: return _REQUEST_MODEL.validate_python(data.get("model"), strict=True) @@ -2703,16 +2714,12 @@ class ProxyBaseLLMRequestProcessing: self.data = independent_snapshot(pristine) self.data["model"] = fallback_model try: - probe_data = { - **_REQUEST_FIELDS.validate_python(pristine), - "model": resolve_served_model( - fallback_model, - _DECODED_JSON.validate_python(litellm.model_alias_map), - key_aliases, - group_alias, - ), - } - if endpoint_unsupported_by_model(route_type, probe_data, llm_router, user_api_key_dict.team_id): + if endpoint_unsupported_by_model( + route_type, + _fallback_probe_data(pristine, fallback_model, key_aliases, group_alias), + llm_router, + user_api_key_dict.team_id, + ): continue return await self.common_processing_pre_call_logic( request=request, diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 357677f8a7b..5da52bd33d7 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1996,6 +1996,7 @@ def _responses_ws_failure_frame(failure: Exception) -> str: return json.dumps( { "type": "error", + "status": 400, "error": {"type": "invalid_request_error", "message": failure.message.split("\n\n", 1)[0]}, } ) @@ -2296,7 +2297,6 @@ async def _responses_websocket_session( user_api_base=user_api_base, model=resolved_model, route_type="_aresponses_websocket", - llm_router=llm_router, skip_guardrails=True, ) except Exception as e: diff --git a/litellm/router_utils/endpoint_eligibility.py b/litellm/router_utils/endpoint_eligibility.py index a1866963e51..018dc07e424 100644 --- a/litellm/router_utils/endpoint_eligibility.py +++ b/litellm/router_utils/endpoint_eligibility.py @@ -4,7 +4,7 @@ from contextvars import ContextVar from types import MappingProxyType from typing import Final, TypeVar -from pydantic import BaseModel, ConfigDict, ValidationError +from pydantic import BaseModel, ConfigDict, ValidationError, field_validator from litellm.exceptions import EndpointNotSupportedByModelError @@ -31,7 +31,8 @@ CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType( def _endpoint_path(endpoint: str) -> str: - return endpoint.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1") + rooted: Final = "/" + endpoint.strip().lstrip("/") + return rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1") CHAT_ENDPOINTS: Final = frozenset(_endpoint_path(endpoint) for endpoint in CHAT_ENDPOINT_BY_CALL.values()) @@ -47,6 +48,11 @@ class _EndpointDeclaration(BaseModel): enforce_supported_endpoints: bool | None = None supported_endpoints: tuple[str, ...] | None = None + @field_validator("supported_endpoints", mode="before") + @classmethod + def _one_endpoint_as_a_list(cls, value: object) -> object: + return (value,) if isinstance(value, str) else value + @contextmanager def endpoint_selection(call_type: object, module: object = None, model: object = None) -> Generator[None]: diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index a88ccf06c44..367135eae26 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -2,7 +2,7 @@ - {id: llm.completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_completions_endpoint_e2e.py", rationale: "Legacy text /completions endpoint, second-highest production request volume"} - {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"} - {id: llm.embeddings.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy ยง9.3 / LIT-4778", rationale: "Missing model/input on /embeddings return client errors"} -- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings and route chat through a live fallback; rejection and zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"} +- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings, refuse chat with a gateway 400, and route chat through a live fallback; zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"} - {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"} - {id: llm.embeddings.openai.passthrough.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "POST /openai_passthrough/v1/embeddings is costed; the route wrote no spend row at all, so budgets never saw traffic OpenAI was billing for (GitHub issue #36646)"} - {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"} diff --git a/tests/e2e/llm_translation/test_embedding_only_supported_endpoints_e2e.py b/tests/e2e/llm_translation/test_embedding_only_supported_endpoints_e2e.py index 899b421b218..c45adfeeb84 100644 --- a/tests/e2e/llm_translation/test_embedding_only_supported_endpoints_e2e.py +++ b/tests/e2e/llm_translation/test_embedding_only_supported_endpoints_e2e.py @@ -1,4 +1,4 @@ -"""Live provider checks for embeddings and chat fallback from an explicitly restricted model.""" +"""Live provider checks for embeddings, the chat refusal, and chat fallback from an explicitly restricted model.""" from __future__ import annotations @@ -10,7 +10,7 @@ from e2e_config import unique_marker from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody, ModelInfoBody, ModelMode, ModelNewBody -from openai import OpenAI +from openai import BadRequestError, OpenAI from proxy_client import ProxyClient from sdk_clients import NO_PROXY_CACHE, SdkClients @@ -102,3 +102,18 @@ class TestEmbeddingOnlySupportedEndpoints: assert completion.choices and completion.choices[0].message.content, ( f"the fallback {fallback} should answer a chat call to {model}, got {completion!r}" ) + + @pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works") + @meta(_subject(Route.CHAT_COMPLETIONS)) + def test_chat_completions_without_a_fallback_are_refused_by_the_gateway( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY) + with pytest.raises(BadRequestError) as refused: + _ = _openai_at(sdk, "/v1", resources.key()).chat.completions.create( + model=model, messages=[{"role": "user", "content": PROMPT}], extra_body=NO_PROXY_CACHE + ) + assert ( + f"/v1/chat/completions is not supported by model={model}. Supported endpoints: /v1/embeddings" + in refused.value.message + ), f"chat on {model} should be refused by the gateway before any provider call, got {refused.value!r}" diff --git a/tests/integration/routing/test_non_chat_model_endpoint_gate.py b/tests/integration/routing/test_non_chat_model_endpoint_gate.py index f6ba0358157..1a5701f370e 100644 --- a/tests/integration/routing/test_non_chat_model_endpoint_gate.py +++ b/tests/integration/routing/test_non_chat_model_endpoint_gate.py @@ -419,6 +419,7 @@ def _websocket_rejection(model: str) -> tuple[dict[str, JsonValue], int | None, return ( { "type": "error", + "status": 400, "error": { "type": "invalid_request_error", "message": f"/v1/responses is not supported by model={model}. Supported endpoints: /v1/embeddings", diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index bacaf8f708f..688f3e64f14 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -1835,6 +1835,7 @@ def test_responses_ws_failure_frame_shows_only_the_refusal_for_a_model_that_refu assert json.loads(_responses_ws_failure_frame(failure)) == { "type": "error", + "status": 400, "error": {"type": "invalid_request_error", "message": refusal}, } @@ -2943,8 +2944,6 @@ class TestResponsesWSFirstFrameModelAuth: ) - - class TestReadWSModelFromFirstFrameErrors: @pytest.mark.asyncio async def test_transport_error_first_frame_closes_with_internal_error(self): diff --git a/tests/unit/router_utils/test_endpoint_eligibility.py b/tests/unit/router_utils/test_endpoint_eligibility.py index a37b4648a36..a208ca946e1 100644 --- a/tests/unit/router_utils/test_endpoint_eligibility.py +++ b/tests/unit/router_utils/test_endpoint_eligibility.py @@ -38,7 +38,7 @@ def test_explicit_restriction_excludes_an_embedding_deployment_from_a_mixed_grou {"enforce_supported_endpoints": False, "supported_endpoints": ["/v1/embeddings"]}, {"enforce_supported_endpoints": True}, {"enforce_supported_endpoints": True, "supported_endpoints": None}, - {"enforce_supported_endpoints": True, "supported_endpoints": "invalid"}, + {"enforce_supported_endpoints": True, "supported_endpoints": 7}, {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/messages"]}, ], ) @@ -49,6 +49,14 @@ def test_advisory_or_incomplete_metadata_preserves_routing(info: Mapping[str, ob assert result == (deployment,) +def test_a_single_endpoint_string_is_enforced_like_a_one_item_list() -> None: + deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": "/v1/embeddings"}) + with endpoint_selection("acompletion"): + result: Final = eligible_deployments("shared", (deployment,)) + assert isinstance(result, EndpointNotSupportedByModelError) + assert result.message.endswith("Supported endpoints: /v1/embeddings") + + def test_unsupported_error_explains_the_requested_operation_and_available_endpoints() -> None: deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]}) with endpoint_selection("acreate_interaction"): @@ -79,7 +87,16 @@ def test_nested_non_chat_calls_do_not_inherit_the_outer_chat_restriction() -> No @pytest.mark.parametrize( "endpoint", - ["/messages", "/interactions", "/responses/compact", "/openai/v1/responses/compact", "/openai/v1/chat/completions"], + [ + "/messages", + "/interactions", + "/responses/compact", + "/openai/v1/responses/compact", + "/openai/v1/chat/completions", + "chat/completions", + "v1/responses", + " /v1/chat/completions ", + ], ) def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint: str) -> None: deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": [endpoint]}) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 41d1b1c0801..2d8199278ea 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -18713,7 +18713,7 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi async def test_anthropic_messages_same_group_retry_skips_an_embedding_only_deployment(): """The retry of a /v1/messages stream dropped before content must not land on an enforced embedding-only deployment of the same group, even when that deployment is the group's preferred order.""" - router = Router( + router: Final = Router( model_list=[ {"model_name": "glm", "litellm_params": {"model": "anthropic/glm-chat", "api_key": "sk-test", "order": 2}}, { @@ -18728,7 +18728,7 @@ async def test_anthropic_messages_same_group_retry_skips_an_embedding_only_deplo _anthropic_messages_dropped_before_content, _anthropic_messages_retried_stream ) - async def anthropic_messages(**kwargs): + async def anthropic_messages(**kwargs: object) -> object: return await provider(**kwargs) stream: Final = await _anthropic_messages_stream_through_router(router, anthropic_messages)