diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 260c8372925..5c31db19f40 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -716,6 +716,18 @@ def get_request_route_template(request: Request) -> str | None: return None +def _safe_get_request_path_params(request: Request | None) -> dict[str, object] | None: + """Starlette's resolved path parameters, or None when unavailable.""" + if request is None: + return None + try: + params: Final = request.path_params + return dict(params) if params else None + except Exception as e: + verbose_proxy_logger.debug("error reading request path_params: %s", e) + return None + + @lru_cache(maxsize=256) def normalize_request_route(route: str) -> str: """ @@ -1697,22 +1709,47 @@ _MODEL_ROUTING_BODY_TARGET_MODEL_ROUTE_MARKERS: Final = ( "/vector_stores", ) _MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS: Final = ("/evals",) -# Routes served by the chat-completions, completions and embeddings handlers. -# These handlers bind a ``model`` path or query parameter -# (``/openai/deployments/{model}/...``, ``/engines/{model}/...``, ``?model=``) -# and route from it in preference to the body ``model``. The allowlist check -# therefore has to validate that value too: with only the body validated, a -# key scoped to one model could name an allowed model in the body and a -# denied one in the query string, and be served the denied model. -_MODEL_ROUTING_LLM_HANDLER_ROUTE_MARKERS: Final = ( - "/chat/completions", - "/completions", - "/embeddings", +# The chat-completions, completions and embeddings handlers (and the queue +# variant) declare a ``model`` parameter that FastAPI binds from the URL — +# from ``?model=`` on the plain routes, from ``{model:path}`` on the +# deployment-style routes — and request processing routes from that value in +# preference to the body ``model``. The allowlist check therefore has to +# validate the URL-bound model too: with only the body validated, a key scoped +# to one model could name an allowed model in the body and a denied one in the +# URL, and be served the denied model. The two sets below are the exact route +# templates those handlers are mounted on (proxy_server.py); classification is +# by matched template, never by substring, so unrelated routes that merely +# contain ``/completions`` do not get a stray ``?model=`` promoted to an +# authorization input. On a ``{model:path}`` route FastAPI binds the path +# value and ignores a same-named query parameter, so only the path value is a +# candidate there. +_LLM_HANDLER_QUERY_MODEL_ROUTES: Final = frozenset( + { + "/v1/chat/completions", + "/chat/completions", + "/queue/chat/completions", + "/v1/completions", + "/completions", + "/v1/embeddings", + "/embeddings", + } ) -# Anchored at the path start on purpose: ``/azure/openai/deployments/...`` is the -# Azure relay route, whose model group is resolved separately below. +_LLM_HANDLER_PATH_MODEL_TEMPLATES: Final = frozenset( + { + "/openai/deployments/{model:path}/chat/completions", + "/engines/{model:path}/chat/completions", + "/openai/deployments/{model:path}/completions", + "/engines/{model:path}/completions", + "/openai/deployments/{model:path}/embeddings", + "/engines/{model:path}/embeddings", + } +) +# Fallback for callers that have no ``Request`` (budget reservation, unit +# tests): recover the ``{model:path}`` value from the literal path. Anchored +# at the path start: ``get_request_route`` already strips ``root_path``, and +# ``/azure/openai/deployments/...`` is the Azure relay, resolved separately. _LLM_HANDLER_PATH_MODEL_PATTERN: Final = re.compile( - r"^/(?:openai/deployments|engines)/(.+?)/(?:chat/completions|completions|embeddings)$" + r"^/(?Popenai/deployments|engines)/(?P.+?)/(?Pchat/completions|completions|embeddings)$" ) # Realtime WebRTC routes carry the effective model inside the nested # ``session.model`` field (see realtime_endpoints.endpoints), so the model the @@ -1777,15 +1814,38 @@ def _route_matches_any_marker(route: str, markers: tuple[str, ...]) -> bool: return any(marker in normalized_route for marker in markers) -def _llm_handler_path_model(route: str) -> str | None: - """Model named in the path of a deployment-style LLM route. +def _llm_handler_url_model( + route: str, + route_template: str | None, + request_path_params: Mapping[str, object] | None, + request_query_params: Mapping[str, object] | None, +) -> str | None: + """The model FastAPI binds from the URL for an LLM handler route, or None. - ``/openai/deployments/{model}/chat/completions`` and ``/engines/{model}/...`` - bind ``{model}`` (which may itself contain ``/``) to the handler's ``model`` - argument, which then takes precedence over the body model when routing. + Mirrors the handlers' ``model`` parameter binding: the ``{model:path}`` + segment on the deployment-style templates (a same-named query parameter + is ignored there, as FastAPI does), the exact-case ``model`` query + parameter on the plain routes (``?MODEL=`` is not bound, so it is not a + candidate; duplicates are last-wins in both FastAPI and the parsed + mapping). ``route_template`` comes from the matched Starlette route when a + request is available; without one, the literal path is classified instead. """ - match: Final = _LLM_HANDLER_PATH_MODEL_PATTERN.match(route) - return match.group(1) if match else None + if route_template is None: + if route in _LLM_HANDLER_QUERY_MODEL_ROUTES: + route_template = route + else: + match: Final = _LLM_HANDLER_PATH_MODEL_PATTERN.match(route) + if match: + route_template = f"/{match.group('prefix')}/{{model:path}}/{match.group('suffix')}" + if request_path_params is None: + request_path_params = {"model": match.group("model")} + if route_template in _LLM_HANDLER_PATH_MODEL_TEMPLATES: + path_model = (request_path_params or {}).get("model") + return path_model if isinstance(path_model, str) and path_model else None + if route_template in _LLM_HANDLER_QUERY_MODEL_ROUTES: + query_model = (request_query_params or {}).get("model") + return query_model if isinstance(query_model, str) and query_model else None + return None def _route_uses_model_routing_sources(route: str) -> bool: @@ -1904,6 +1964,8 @@ def _extract_model_candidates_from_request( request_query_params: Mapping[str, object] | None = None, llm_router: Router | None = None, team_id: str | None = None, + route_template: str | None = None, + request_path_params: Mapping[str, object] | None = None, ) -> list[str]: if route == "/cost/predict-cache": prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload @@ -1933,13 +1995,19 @@ def _extract_model_candidates_from_request( _append_model_candidates(candidates, session.get("model")) if uses_completion_model_sources and isinstance(request_data.get("completion"), dict): _append_model_candidates(candidates, request_data["completion"].get("model")) - if _route_matches_any_marker(route=route, markers=_MODEL_ROUTING_LLM_HANDLER_ROUTE_MARKERS): - # The handler routes from ``?model=`` / ``{model}`` ahead of the body, so - # every model it can end up dispatching to must pass the allowlist. The - # ``x-litellm-model`` header is deliberately not read here: these handlers - # do not route from it. - _append_model_candidates(candidates, _get_case_insensitive_mapping_value(request_query_params, "model")) - _append_model_candidates(candidates, _llm_handler_path_model(route)) + # URL-bound model on the LLM handler routes: request processing prefers it + # over the body model, so it must pass the allowlist as well. The + # ``x-litellm-model`` header is deliberately not read: these handlers do + # not route from it. + _append_model_candidates( + candidates, + _llm_handler_url_model( + route=route, + route_template=route_template, + request_path_params=request_path_params, + request_query_params=request_query_params, + ), + ) if uses_model_routing_sources: if uses_header_or_query_model_sources: @@ -2037,6 +2105,8 @@ def get_model_from_request( request_query_params=request_query_params, llm_router=llm_router, team_id=team_id, + route_template=get_request_route_template(request) if request is not None else None, + request_path_params=_safe_get_request_path_params(request), ) model = _format_model_candidates(candidates) diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 540c0d620db..2df1d1529e1 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -3919,11 +3919,22 @@ class TestIsRequestBodySafeBlocksAwsIdentitySelectors: is True ) -# --- query-string / path model must be allowlisted on LLM handler routes ------------ -# The chat-completions, completions and embeddings handlers route from a ``model`` -# query or path parameter in preference to the body model. Only the body used to -# be validated, so a key scoped to one model could be served another by putting -# it in ``?model=`` (BerriAI/litellm#41810). +# --- URL-bound model must be allowlisted on the LLM handler routes ------------------- +# chat-completions, completions and embeddings bind ``model`` from the URL +# (``?model=`` on the plain routes, ``{model:path}`` on the deployment-style +# routes) and route from it ahead of the body model. Only the body used to be +# validated, so a key scoped to one model could be served another +# (BerriAI/litellm#41810). + + +def _matched_request(template: str, path_params: dict | None = None) -> Request: + """Request as Starlette leaves it after routing: ``scope["route"]`` carries the + matched template and ``path_params`` the resolved segments.""" + + class _Route: + path = template + + return Request(scope={"type": "http", "headers": [], "route": _Route(), "path_params": path_params or {}}) @pytest.mark.parametrize( @@ -3931,6 +3942,7 @@ class TestIsRequestBodySafeBlocksAwsIdentitySelectors: [ "/v1/chat/completions", "/chat/completions", + "/queue/chat/completions", "/v1/completions", "/completions", "/v1/embeddings", @@ -3944,6 +3956,16 @@ def test_get_model_from_request_includes_query_model_on_llm_handler_routes(route request_data={"model": "allowed-model"}, route=route, request_query_params={"model": "denied-model"}, + request=_matched_request(route), + ) == ["allowed-model", "denied-model"] + + +def test_get_model_from_request_query_model_without_request_object(): + """Callers with no Request (budget reservation) classify the literal path.""" + assert get_model_from_request( + request_data={"model": "allowed-model"}, + route="/v1/chat/completions", + request_query_params={"model": "denied-model"}, ) == ["allowed-model", "denied-model"] @@ -3953,6 +3975,7 @@ def test_get_model_from_request_query_model_alone_on_chat_completions(): request_data={"messages": []}, route="/v1/chat/completions", request_query_params={"model": "denied-model"}, + request=_matched_request("/v1/chat/completions"), ) == "denied-model" ) @@ -3964,37 +3987,110 @@ def test_get_model_from_request_query_model_equal_to_body_dedupes(): request_data={"model": "allowed-model"}, route="/v1/chat/completions", request_query_params={"model": "allowed-model"}, + request=_matched_request("/v1/chat/completions"), + ) + == "allowed-model" + ) + + +def test_get_model_from_request_query_model_lookup_is_exact_case(): + """FastAPI binds ``model`` case-sensitively: ``?MODEL=`` is not bound by the + handler, so it must not become a candidate either (it would only false-403).""" + assert ( + get_model_from_request( + request_data={"model": "allowed-model"}, + route="/v1/chat/completions", + request_query_params={"MODEL": "denied-model"}, + request=_matched_request("/v1/chat/completions"), ) == "allowed-model" ) @pytest.mark.parametrize( - "route, expected", + "template, path, path_model", [ - ("/openai/deployments/denied-model/chat/completions", ["allowed-model", "denied-model"]), - ("/openai/deployments/denied-model/completions", ["allowed-model", "denied-model"]), - ("/openai/deployments/denied-model/embeddings", ["allowed-model", "denied-model"]), - ("/engines/denied-model/chat/completions", ["allowed-model", "denied-model"]), + ("/openai/deployments/{model:path}/chat/completions", "/openai/deployments/denied-model/chat/completions", "denied-model"), + ("/openai/deployments/{model:path}/completions", "/openai/deployments/denied-model/completions", "denied-model"), + ("/openai/deployments/{model:path}/embeddings", "/openai/deployments/denied-model/embeddings", "denied-model"), + ("/engines/{model:path}/chat/completions", "/engines/denied-model/chat/completions", "denied-model"), + ("/engines/{model:path}/completions", "/engines/denied-model/completions", "denied-model"), + ("/engines/{model:path}/embeddings", "/engines/denied-model/embeddings", "denied-model"), # {model:path} may contain slashes - ("/openai/deployments/org/denied-model/chat/completions", ["allowed-model", "org/denied-model"]), + ("/openai/deployments/{model:path}/chat/completions", "/openai/deployments/org/denied-model/chat/completions", "org/denied-model"), ], ) -def test_get_model_from_request_includes_deployment_path_model(route, expected): - """The ``{model}`` path segment is bound to the handler and routed from even when +def test_get_model_from_request_includes_deployment_path_model(template, path, path_model): + """The ``{model:path}`` segment is bound to the handler and routed from even when the body carries a different model, so it is validated alongside the body.""" - assert get_model_from_request(request_data={"model": "allowed-model"}, route=route) == expected + assert get_model_from_request( + request_data={"model": "allowed-model"}, + route=path, + request=_matched_request(template, {"model": path_model}), + ) == ["allowed-model", path_model] -@pytest.mark.parametrize("route", ["/v1/messages", "/v1/responses"]) -def test_get_model_from_request_ignores_query_model_on_routes_that_do_not_route_from_it(route): - """/v1/messages and /v1/responses serve the body model regardless of ``?model=``, - so the query value is not a candidate there (unchanged behaviour).""" +def test_get_model_from_request_deployment_path_model_without_request_object(): + """No Request: the literal path is parsed, slashes in the model included.""" + assert get_model_from_request( + request_data={"model": "allowed-model"}, + route="/openai/deployments/org/denied-model/chat/completions", + ) == ["allowed-model", "org/denied-model"] + + +def test_get_model_from_request_deployment_route_ignores_query_model(): + """On a ``{model:path}`` route FastAPI binds the path value and ignores a + same-named query parameter, so auth validates exactly what will be routed.""" + assert ( + get_model_from_request( + request_data={"model": "allowed-model"}, + route="/openai/deployments/allowed-model/chat/completions", + request_query_params={"model": "denied-model"}, + request=_matched_request("/openai/deployments/{model:path}/chat/completions", {"model": "allowed-model"}), + ) + == "allowed-model" + ) + + +def test_get_model_from_request_prefers_resolved_path_params_over_literal_path(): + """The value FastAPI resolved wins over re-parsing the path (encoding, slashes).""" + assert get_model_from_request( + request_data={"model": "allowed-model"}, + route="/openai/deployments/a%2Fb/chat/completions", + request=_matched_request("/openai/deployments/{model:path}/chat/completions", {"model": "a/b"}), + ) == ["allowed-model", "a/b"] + + +@pytest.mark.parametrize( + "route, template", + [ + ("/v1/messages", "/v1/messages"), + ("/v1/responses", "/v1/responses"), + ("/cursor/chat/completions", "/cursor/chat/completions"), + ], +) +def test_get_model_from_request_ignores_query_model_on_routes_that_do_not_bind_it(route, template): + """Routes whose handler does not bind ``?model=`` (including ones that merely + contain ``/chat/completions`` in the path) keep serving the body model, so the + query value is not a candidate there.""" assert ( get_model_from_request( request_data={"model": "allowed-model"}, route=route, request_query_params={"model": "denied-model"}, + request=_matched_request(template), + ) + == "allowed-model" + ) + + +def test_get_model_from_request_azure_relay_path_is_not_treated_as_deployment_route(): + """``/azure/openai/deployments/...`` is the Azure relay with its own resolver; + without a Request it must not be parsed as a plain deployment route either.""" + assert ( + get_model_from_request( + request_data={"model": "allowed-model"}, + route="/azure/openai/deployments/denied-model/chat/completions", ) == "allowed-model" ) @@ -4008,6 +4104,7 @@ def test_get_model_from_request_ignores_routing_header_on_chat_completions(): request_data={"model": "allowed-model"}, route="/v1/chat/completions", request_headers={"x-litellm-model": "denied-model"}, + request=_matched_request("/v1/chat/completions"), ) == "allowed-model" )