mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): keep key router settings off the Responses WebSocket, read unrooted or single-string endpoint lists, and add status to the refusal frame
This commit is contained in:
parent
081dd18158
commit
f7d35756bf
9 changed files with 67 additions and 22 deletions
|
|
@ -872,6 +872,17 @@ _DECODED_JSON: Final = TypeAdapter(object)
|
|||
_ROUTER_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
|
||||
|
||||
def _fallback_probe_data(
|
||||
pristine: object, fallback_model: str, key_aliases: object, group_alias: object
|
||||
) -> Mapping[str, object]:
|
||||
return {
|
||||
**_REQUEST_FIELDS.validate_python(pristine),
|
||||
"model": resolve_served_model(
|
||||
fallback_model, _DECODED_JSON.validate_python(litellm.model_alias_map), key_aliases, group_alias
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _request_model(data: Mapping[str, object]) -> str | list[str] | None:
|
||||
try:
|
||||
return _REQUEST_MODEL.validate_python(data.get("model"), strict=True)
|
||||
|
|
@ -2703,16 +2714,12 @@ class ProxyBaseLLMRequestProcessing:
|
|||
self.data = independent_snapshot(pristine)
|
||||
self.data["model"] = fallback_model
|
||||
try:
|
||||
probe_data = {
|
||||
**_REQUEST_FIELDS.validate_python(pristine),
|
||||
"model": resolve_served_model(
|
||||
fallback_model,
|
||||
_DECODED_JSON.validate_python(litellm.model_alias_map),
|
||||
key_aliases,
|
||||
group_alias,
|
||||
),
|
||||
}
|
||||
if endpoint_unsupported_by_model(route_type, probe_data, llm_router, user_api_key_dict.team_id):
|
||||
if endpoint_unsupported_by_model(
|
||||
route_type,
|
||||
_fallback_probe_data(pristine, fallback_model, key_aliases, group_alias),
|
||||
llm_router,
|
||||
user_api_key_dict.team_id,
|
||||
):
|
||||
continue
|
||||
return await self.common_processing_pre_call_logic(
|
||||
request=request,
|
||||
|
|
|
|||
|
|
@ -1996,6 +1996,7 @@ def _responses_ws_failure_frame(failure: Exception) -> str:
|
|||
return json.dumps(
|
||||
{
|
||||
"type": "error",
|
||||
"status": 400,
|
||||
"error": {"type": "invalid_request_error", "message": failure.message.split("\n\n", 1)[0]},
|
||||
}
|
||||
)
|
||||
|
|
@ -2296,7 +2297,6 @@ async def _responses_websocket_session(
|
|||
user_api_base=user_api_base,
|
||||
model=resolved_model,
|
||||
route_type="_aresponses_websocket",
|
||||
llm_router=llm_router,
|
||||
skip_guardrails=True,
|
||||
)
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from contextvars import ContextVar
|
|||
from types import MappingProxyType
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
|
||||
|
||||
from litellm.exceptions import EndpointNotSupportedByModelError
|
||||
|
||||
|
|
@ -31,7 +31,8 @@ CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
|
|||
|
||||
|
||||
def _endpoint_path(endpoint: str) -> str:
|
||||
return endpoint.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
|
||||
rooted: Final = "/" + endpoint.strip().lstrip("/")
|
||||
return rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
|
||||
|
||||
|
||||
CHAT_ENDPOINTS: Final = frozenset(_endpoint_path(endpoint) for endpoint in CHAT_ENDPOINT_BY_CALL.values())
|
||||
|
|
@ -47,6 +48,11 @@ class _EndpointDeclaration(BaseModel):
|
|||
enforce_supported_endpoints: bool | None = None
|
||||
supported_endpoints: tuple[str, ...] | None = None
|
||||
|
||||
@field_validator("supported_endpoints", mode="before")
|
||||
@classmethod
|
||||
def _one_endpoint_as_a_list(cls, value: object) -> object:
|
||||
return (value,) if isinstance(value, str) else value
|
||||
|
||||
|
||||
@contextmanager
|
||||
def endpoint_selection(call_type: object, module: object = None, model: object = None) -> Generator[None]:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
- {id: llm.completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_completions_endpoint_e2e.py", rationale: "Legacy text /completions endpoint, second-highest production request volume"}
|
||||
- {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"}
|
||||
- {id: llm.embeddings.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.3 / LIT-4778", rationale: "Missing model/input on /embeddings return client errors"}
|
||||
- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings and route chat through a live fallback; rejection and zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"}
|
||||
- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings, refuse chat with a gateway 400, and route chat through a live fallback; zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"}
|
||||
- {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"}
|
||||
- {id: llm.embeddings.openai.passthrough.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "POST /openai_passthrough/v1/embeddings is costed; the route wrote no spend row at all, so budgets never saw traffic OpenAI was billing for (GitHub issue #36646)"}
|
||||
- {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Live provider checks for embeddings and chat fallback from an explicitly restricted model."""
|
||||
"""Live provider checks for embeddings, the chat refusal, and chat fallback from an explicitly restricted model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
|
@ -10,7 +10,7 @@ from e2e_config import unique_marker
|
|||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, ModelInfoBody, ModelMode, ModelNewBody
|
||||
from openai import OpenAI
|
||||
from openai import BadRequestError, OpenAI
|
||||
from proxy_client import ProxyClient
|
||||
from sdk_clients import NO_PROXY_CACHE, SdkClients
|
||||
|
||||
|
|
@ -102,3 +102,18 @@ class TestEmbeddingOnlySupportedEndpoints:
|
|||
assert completion.choices and completion.choices[0].message.content, (
|
||||
f"the fallback {fallback} should answer a chat call to {model}, got {completion!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
|
||||
@meta(_subject(Route.CHAT_COMPLETIONS))
|
||||
def test_chat_completions_without_a_fallback_are_refused_by_the_gateway(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
|
||||
with pytest.raises(BadRequestError) as refused:
|
||||
_ = _openai_at(sdk, "/v1", resources.key()).chat.completions.create(
|
||||
model=model, messages=[{"role": "user", "content": PROMPT}], extra_body=NO_PROXY_CACHE
|
||||
)
|
||||
assert (
|
||||
f"/v1/chat/completions is not supported by model={model}. Supported endpoints: /v1/embeddings"
|
||||
in refused.value.message
|
||||
), f"chat on {model} should be refused by the gateway before any provider call, got {refused.value!r}"
|
||||
|
|
|
|||
|
|
@ -419,6 +419,7 @@ def _websocket_rejection(model: str) -> tuple[dict[str, JsonValue], int | None,
|
|||
return (
|
||||
{
|
||||
"type": "error",
|
||||
"status": 400,
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"message": f"/v1/responses is not supported by model={model}. Supported endpoints: /v1/embeddings",
|
||||
|
|
|
|||
|
|
@ -1835,6 +1835,7 @@ def test_responses_ws_failure_frame_shows_only_the_refusal_for_a_model_that_refu
|
|||
|
||||
assert json.loads(_responses_ws_failure_frame(failure)) == {
|
||||
"type": "error",
|
||||
"status": 400,
|
||||
"error": {"type": "invalid_request_error", "message": refusal},
|
||||
}
|
||||
|
||||
|
|
@ -2943,8 +2944,6 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
)
|
||||
|
||||
|
||||
|
||||
|
||||
class TestReadWSModelFromFirstFrameErrors:
|
||||
@pytest.mark.asyncio
|
||||
async def test_transport_error_first_frame_closes_with_internal_error(self):
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ def test_explicit_restriction_excludes_an_embedding_deployment_from_a_mixed_grou
|
|||
{"enforce_supported_endpoints": False, "supported_endpoints": ["/v1/embeddings"]},
|
||||
{"enforce_supported_endpoints": True},
|
||||
{"enforce_supported_endpoints": True, "supported_endpoints": None},
|
||||
{"enforce_supported_endpoints": True, "supported_endpoints": "invalid"},
|
||||
{"enforce_supported_endpoints": True, "supported_endpoints": 7},
|
||||
{"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/messages"]},
|
||||
],
|
||||
)
|
||||
|
|
@ -49,6 +49,14 @@ def test_advisory_or_incomplete_metadata_preserves_routing(info: Mapping[str, ob
|
|||
assert result == (deployment,)
|
||||
|
||||
|
||||
def test_a_single_endpoint_string_is_enforced_like_a_one_item_list() -> None:
|
||||
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": "/v1/embeddings"})
|
||||
with endpoint_selection("acompletion"):
|
||||
result: Final = eligible_deployments("shared", (deployment,))
|
||||
assert isinstance(result, EndpointNotSupportedByModelError)
|
||||
assert result.message.endswith("Supported endpoints: /v1/embeddings")
|
||||
|
||||
|
||||
def test_unsupported_error_explains_the_requested_operation_and_available_endpoints() -> None:
|
||||
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]})
|
||||
with endpoint_selection("acreate_interaction"):
|
||||
|
|
@ -79,7 +87,16 @@ def test_nested_non_chat_calls_do_not_inherit_the_outer_chat_restriction() -> No
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint",
|
||||
["/messages", "/interactions", "/responses/compact", "/openai/v1/responses/compact", "/openai/v1/chat/completions"],
|
||||
[
|
||||
"/messages",
|
||||
"/interactions",
|
||||
"/responses/compact",
|
||||
"/openai/v1/responses/compact",
|
||||
"/openai/v1/chat/completions",
|
||||
"chat/completions",
|
||||
"v1/responses",
|
||||
" /v1/chat/completions ",
|
||||
],
|
||||
)
|
||||
def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint: str) -> None:
|
||||
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": [endpoint]})
|
||||
|
|
|
|||
|
|
@ -18713,7 +18713,7 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi
|
|||
async def test_anthropic_messages_same_group_retry_skips_an_embedding_only_deployment():
|
||||
"""The retry of a /v1/messages stream dropped before content must not land on an enforced embedding-only
|
||||
deployment of the same group, even when that deployment is the group's preferred order."""
|
||||
router = Router(
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "glm", "litellm_params": {"model": "anthropic/glm-chat", "api_key": "sk-test", "order": 2}},
|
||||
{
|
||||
|
|
@ -18728,7 +18728,7 @@ async def test_anthropic_messages_same_group_retry_skips_an_embedding_only_deplo
|
|||
_anthropic_messages_dropped_before_content, _anthropic_messages_retried_stream
|
||||
)
|
||||
|
||||
async def anthropic_messages(**kwargs):
|
||||
async def anthropic_messages(**kwargs: object) -> object:
|
||||
return await provider(**kwargs)
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, anthropic_messages)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue