fix(proxy): keep key router settings off the Responses WebSocket, read unrooted or single-string endpoint lists, and add status to the refusal frame

This commit is contained in:
mateo-berri 2026-10-29 01:04:43 -07:00
parent 081dd18158
commit f7d35756bf
9 changed files with 67 additions and 22 deletions

View file

@ -872,6 +872,17 @@ _DECODED_JSON: Final = TypeAdapter(object)
_ROUTER_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
def _fallback_probe_data(
pristine: object, fallback_model: str, key_aliases: object, group_alias: object
) -> Mapping[str, object]:
return {
**_REQUEST_FIELDS.validate_python(pristine),
"model": resolve_served_model(
fallback_model, _DECODED_JSON.validate_python(litellm.model_alias_map), key_aliases, group_alias
),
}
def _request_model(data: Mapping[str, object]) -> str | list[str] | None:
try:
return _REQUEST_MODEL.validate_python(data.get("model"), strict=True)
@ -2703,16 +2714,12 @@ class ProxyBaseLLMRequestProcessing:
self.data = independent_snapshot(pristine)
self.data["model"] = fallback_model
try:
probe_data = {
**_REQUEST_FIELDS.validate_python(pristine),
"model": resolve_served_model(
fallback_model,
_DECODED_JSON.validate_python(litellm.model_alias_map),
key_aliases,
group_alias,
),
}
if endpoint_unsupported_by_model(route_type, probe_data, llm_router, user_api_key_dict.team_id):
if endpoint_unsupported_by_model(
route_type,
_fallback_probe_data(pristine, fallback_model, key_aliases, group_alias),
llm_router,
user_api_key_dict.team_id,
):
continue
return await self.common_processing_pre_call_logic(
request=request,

View file

@ -1996,6 +1996,7 @@ def _responses_ws_failure_frame(failure: Exception) -> str:
return json.dumps(
{
"type": "error",
"status": 400,
"error": {"type": "invalid_request_error", "message": failure.message.split("\n\n", 1)[0]},
}
)
@ -2296,7 +2297,6 @@ async def _responses_websocket_session(
user_api_base=user_api_base,
model=resolved_model,
route_type="_aresponses_websocket",
llm_router=llm_router,
skip_guardrails=True,
)
except Exception as e:

View file

@ -4,7 +4,7 @@ from contextvars import ContextVar
from types import MappingProxyType
from typing import Final, TypeVar
from pydantic import BaseModel, ConfigDict, ValidationError
from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
from litellm.exceptions import EndpointNotSupportedByModelError
@ -31,7 +31,8 @@ CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
def _endpoint_path(endpoint: str) -> str:
return endpoint.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
rooted: Final = "/" + endpoint.strip().lstrip("/")
return rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
CHAT_ENDPOINTS: Final = frozenset(_endpoint_path(endpoint) for endpoint in CHAT_ENDPOINT_BY_CALL.values())
@ -47,6 +48,11 @@ class _EndpointDeclaration(BaseModel):
enforce_supported_endpoints: bool | None = None
supported_endpoints: tuple[str, ...] | None = None
@field_validator("supported_endpoints", mode="before")
@classmethod
def _one_endpoint_as_a_list(cls, value: object) -> object:
return (value,) if isinstance(value, str) else value
@contextmanager
def endpoint_selection(call_type: object, module: object = None, model: object = None) -> Generator[None]:

View file

@ -2,7 +2,7 @@
- {id: llm.completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_completions_endpoint_e2e.py", rationale: "Legacy text /completions endpoint, second-highest production request volume"}
- {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"}
- {id: llm.embeddings.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.3 / LIT-4778", rationale: "Missing model/input on /embeddings return client errors"}
- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings and route chat through a live fallback; rejection and zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"}
- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings, refuse chat with a gateway 400, and route chat through a live fallback; zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"}
- {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"}
- {id: llm.embeddings.openai.passthrough.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "POST /openai_passthrough/v1/embeddings is costed; the route wrote no spend row at all, so budgets never saw traffic OpenAI was billing for (GitHub issue #36646)"}
- {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"}

View file

@ -1,4 +1,4 @@
"""Live provider checks for embeddings and chat fallback from an explicitly restricted model."""
"""Live provider checks for embeddings, the chat refusal, and chat fallback from an explicitly restricted model."""
from __future__ import annotations
@ -10,7 +10,7 @@ from e2e_config import unique_marker
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody, ModelInfoBody, ModelMode, ModelNewBody
from openai import OpenAI
from openai import BadRequestError, OpenAI
from proxy_client import ProxyClient
from sdk_clients import NO_PROXY_CACHE, SdkClients
@ -102,3 +102,18 @@ class TestEmbeddingOnlySupportedEndpoints:
assert completion.choices and completion.choices[0].message.content, (
f"the fallback {fallback} should answer a chat call to {model}, got {completion!r}"
)
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
@meta(_subject(Route.CHAT_COMPLETIONS))
def test_chat_completions_without_a_fallback_are_refused_by_the_gateway(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
with pytest.raises(BadRequestError) as refused:
_ = _openai_at(sdk, "/v1", resources.key()).chat.completions.create(
model=model, messages=[{"role": "user", "content": PROMPT}], extra_body=NO_PROXY_CACHE
)
assert (
f"/v1/chat/completions is not supported by model={model}. Supported endpoints: /v1/embeddings"
in refused.value.message
), f"chat on {model} should be refused by the gateway before any provider call, got {refused.value!r}"

View file

@ -419,6 +419,7 @@ def _websocket_rejection(model: str) -> tuple[dict[str, JsonValue], int | None,
return (
{
"type": "error",
"status": 400,
"error": {
"type": "invalid_request_error",
"message": f"/v1/responses is not supported by model={model}. Supported endpoints: /v1/embeddings",

View file

@ -1835,6 +1835,7 @@ def test_responses_ws_failure_frame_shows_only_the_refusal_for_a_model_that_refu
assert json.loads(_responses_ws_failure_frame(failure)) == {
"type": "error",
"status": 400,
"error": {"type": "invalid_request_error", "message": refusal},
}
@ -2943,8 +2944,6 @@ class TestResponsesWSFirstFrameModelAuth:
)
class TestReadWSModelFromFirstFrameErrors:
@pytest.mark.asyncio
async def test_transport_error_first_frame_closes_with_internal_error(self):

View file

@ -38,7 +38,7 @@ def test_explicit_restriction_excludes_an_embedding_deployment_from_a_mixed_grou
{"enforce_supported_endpoints": False, "supported_endpoints": ["/v1/embeddings"]},
{"enforce_supported_endpoints": True},
{"enforce_supported_endpoints": True, "supported_endpoints": None},
{"enforce_supported_endpoints": True, "supported_endpoints": "invalid"},
{"enforce_supported_endpoints": True, "supported_endpoints": 7},
{"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/messages"]},
],
)
@ -49,6 +49,14 @@ def test_advisory_or_incomplete_metadata_preserves_routing(info: Mapping[str, ob
assert result == (deployment,)
def test_a_single_endpoint_string_is_enforced_like_a_one_item_list() -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": "/v1/embeddings"})
with endpoint_selection("acompletion"):
result: Final = eligible_deployments("shared", (deployment,))
assert isinstance(result, EndpointNotSupportedByModelError)
assert result.message.endswith("Supported endpoints: /v1/embeddings")
def test_unsupported_error_explains_the_requested_operation_and_available_endpoints() -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]})
with endpoint_selection("acreate_interaction"):
@ -79,7 +87,16 @@ def test_nested_non_chat_calls_do_not_inherit_the_outer_chat_restriction() -> No
@pytest.mark.parametrize(
"endpoint",
["/messages", "/interactions", "/responses/compact", "/openai/v1/responses/compact", "/openai/v1/chat/completions"],
[
"/messages",
"/interactions",
"/responses/compact",
"/openai/v1/responses/compact",
"/openai/v1/chat/completions",
"chat/completions",
"v1/responses",
" /v1/chat/completions ",
],
)
def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint: str) -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": [endpoint]})

View file

@ -18713,7 +18713,7 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi
async def test_anthropic_messages_same_group_retry_skips_an_embedding_only_deployment():
"""The retry of a /v1/messages stream dropped before content must not land on an enforced embedding-only
deployment of the same group, even when that deployment is the group's preferred order."""
router = Router(
router: Final = Router(
model_list=[
{"model_name": "glm", "litellm_params": {"model": "anthropic/glm-chat", "api_key": "sk-test", "order": 2}},
{
@ -18728,7 +18728,7 @@ async def test_anthropic_messages_same_group_retry_skips_an_embedding_only_deplo
_anthropic_messages_dropped_before_content, _anthropic_messages_retried_stream
)
async def anthropic_messages(**kwargs):
async def anthropic_messages(**kwargs: object) -> object:
return await provider(**kwargs)
stream: Final = await _anthropic_messages_stream_through_router(router, anthropic_messages)