mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): classify by matched route template and resolved path params for the URL-bound model check
Review follow-up: replace substring route markers with the exact set of
route templates the chat/completions/embeddings (and queue) handlers are
mounted on, read the `{model:path}` value from Starlette's resolved
path_params instead of re-parsing the path, ignore a same-named query
parameter on path routes (FastAPI binds the path value), and look the query
parameter up exact-case (`?MODEL=` is not bound). The literal-path fallback
remains only for callers without a Request.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Hs4zF96YHUSUHnCjAVM1xD
This commit is contained in:
parent
9eaf2eb3c9
commit
caed61bda1
2 changed files with 213 additions and 46 deletions
|
|
@ -716,6 +716,18 @@ def get_request_route_template(request: Request) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _safe_get_request_path_params(request: Request | None) -> dict[str, object] | None:
|
||||
"""Starlette's resolved path parameters, or None when unavailable."""
|
||||
if request is None:
|
||||
return None
|
||||
try:
|
||||
params: Final = request.path_params
|
||||
return dict(params) if params else None
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug("error reading request path_params: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def normalize_request_route(route: str) -> str:
|
||||
"""
|
||||
|
|
@ -1697,22 +1709,47 @@ _MODEL_ROUTING_BODY_TARGET_MODEL_ROUTE_MARKERS: Final = (
|
|||
"/vector_stores",
|
||||
)
|
||||
_MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS: Final = ("/evals",)
|
||||
# Routes served by the chat-completions, completions and embeddings handlers.
|
||||
# These handlers bind a ``model`` path or query parameter
|
||||
# (``/openai/deployments/{model}/...``, ``/engines/{model}/...``, ``?model=``)
|
||||
# and route from it in preference to the body ``model``. The allowlist check
|
||||
# therefore has to validate that value too: with only the body validated, a
|
||||
# key scoped to one model could name an allowed model in the body and a
|
||||
# denied one in the query string, and be served the denied model.
|
||||
_MODEL_ROUTING_LLM_HANDLER_ROUTE_MARKERS: Final = (
|
||||
"/chat/completions",
|
||||
"/completions",
|
||||
"/embeddings",
|
||||
# The chat-completions, completions and embeddings handlers (and the queue
|
||||
# variant) declare a ``model`` parameter that FastAPI binds from the URL —
|
||||
# from ``?model=`` on the plain routes, from ``{model:path}`` on the
|
||||
# deployment-style routes — and request processing routes from that value in
|
||||
# preference to the body ``model``. The allowlist check therefore has to
|
||||
# validate the URL-bound model too: with only the body validated, a key scoped
|
||||
# to one model could name an allowed model in the body and a denied one in the
|
||||
# URL, and be served the denied model. The two sets below are the exact route
|
||||
# templates those handlers are mounted on (proxy_server.py); classification is
|
||||
# by matched template, never by substring, so unrelated routes that merely
|
||||
# contain ``/completions`` do not get a stray ``?model=`` promoted to an
|
||||
# authorization input. On a ``{model:path}`` route FastAPI binds the path
|
||||
# value and ignores a same-named query parameter, so only the path value is a
|
||||
# candidate there.
|
||||
_LLM_HANDLER_QUERY_MODEL_ROUTES: Final = frozenset(
|
||||
{
|
||||
"/v1/chat/completions",
|
||||
"/chat/completions",
|
||||
"/queue/chat/completions",
|
||||
"/v1/completions",
|
||||
"/completions",
|
||||
"/v1/embeddings",
|
||||
"/embeddings",
|
||||
}
|
||||
)
|
||||
# Anchored at the path start on purpose: ``/azure/openai/deployments/...`` is the
|
||||
# Azure relay route, whose model group is resolved separately below.
|
||||
_LLM_HANDLER_PATH_MODEL_TEMPLATES: Final = frozenset(
|
||||
{
|
||||
"/openai/deployments/{model:path}/chat/completions",
|
||||
"/engines/{model:path}/chat/completions",
|
||||
"/openai/deployments/{model:path}/completions",
|
||||
"/engines/{model:path}/completions",
|
||||
"/openai/deployments/{model:path}/embeddings",
|
||||
"/engines/{model:path}/embeddings",
|
||||
}
|
||||
)
|
||||
# Fallback for callers that have no ``Request`` (budget reservation, unit
|
||||
# tests): recover the ``{model:path}`` value from the literal path. Anchored
|
||||
# at the path start: ``get_request_route`` already strips ``root_path``, and
|
||||
# ``/azure/openai/deployments/...`` is the Azure relay, resolved separately.
|
||||
_LLM_HANDLER_PATH_MODEL_PATTERN: Final = re.compile(
|
||||
r"^/(?:openai/deployments|engines)/(.+?)/(?:chat/completions|completions|embeddings)$"
|
||||
r"^/(?P<prefix>openai/deployments|engines)/(?P<model>.+?)/(?P<suffix>chat/completions|completions|embeddings)$"
|
||||
)
|
||||
# Realtime WebRTC routes carry the effective model inside the nested
|
||||
# ``session.model`` field (see realtime_endpoints.endpoints), so the model the
|
||||
|
|
@ -1777,15 +1814,38 @@ def _route_matches_any_marker(route: str, markers: tuple[str, ...]) -> bool:
|
|||
return any(marker in normalized_route for marker in markers)
|
||||
|
||||
|
||||
def _llm_handler_path_model(route: str) -> str | None:
|
||||
"""Model named in the path of a deployment-style LLM route.
|
||||
def _llm_handler_url_model(
|
||||
route: str,
|
||||
route_template: str | None,
|
||||
request_path_params: Mapping[str, object] | None,
|
||||
request_query_params: Mapping[str, object] | None,
|
||||
) -> str | None:
|
||||
"""The model FastAPI binds from the URL for an LLM handler route, or None.
|
||||
|
||||
``/openai/deployments/{model}/chat/completions`` and ``/engines/{model}/...``
|
||||
bind ``{model}`` (which may itself contain ``/``) to the handler's ``model``
|
||||
argument, which then takes precedence over the body model when routing.
|
||||
Mirrors the handlers' ``model`` parameter binding: the ``{model:path}``
|
||||
segment on the deployment-style templates (a same-named query parameter
|
||||
is ignored there, as FastAPI does), the exact-case ``model`` query
|
||||
parameter on the plain routes (``?MODEL=`` is not bound, so it is not a
|
||||
candidate; duplicates are last-wins in both FastAPI and the parsed
|
||||
mapping). ``route_template`` comes from the matched Starlette route when a
|
||||
request is available; without one, the literal path is classified instead.
|
||||
"""
|
||||
match: Final = _LLM_HANDLER_PATH_MODEL_PATTERN.match(route)
|
||||
return match.group(1) if match else None
|
||||
if route_template is None:
|
||||
if route in _LLM_HANDLER_QUERY_MODEL_ROUTES:
|
||||
route_template = route
|
||||
else:
|
||||
match: Final = _LLM_HANDLER_PATH_MODEL_PATTERN.match(route)
|
||||
if match:
|
||||
route_template = f"/{match.group('prefix')}/{{model:path}}/{match.group('suffix')}"
|
||||
if request_path_params is None:
|
||||
request_path_params = {"model": match.group("model")}
|
||||
if route_template in _LLM_HANDLER_PATH_MODEL_TEMPLATES:
|
||||
path_model = (request_path_params or {}).get("model")
|
||||
return path_model if isinstance(path_model, str) and path_model else None
|
||||
if route_template in _LLM_HANDLER_QUERY_MODEL_ROUTES:
|
||||
query_model = (request_query_params or {}).get("model")
|
||||
return query_model if isinstance(query_model, str) and query_model else None
|
||||
return None
|
||||
|
||||
|
||||
def _route_uses_model_routing_sources(route: str) -> bool:
|
||||
|
|
@ -1904,6 +1964,8 @@ def _extract_model_candidates_from_request(
|
|||
request_query_params: Mapping[str, object] | None = None,
|
||||
llm_router: Router | None = None,
|
||||
team_id: str | None = None,
|
||||
route_template: str | None = None,
|
||||
request_path_params: Mapping[str, object] | None = None,
|
||||
) -> list[str]:
|
||||
if route == "/cost/predict-cache":
|
||||
prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload
|
||||
|
|
@ -1933,13 +1995,19 @@ def _extract_model_candidates_from_request(
|
|||
_append_model_candidates(candidates, session.get("model"))
|
||||
if uses_completion_model_sources and isinstance(request_data.get("completion"), dict):
|
||||
_append_model_candidates(candidates, request_data["completion"].get("model"))
|
||||
if _route_matches_any_marker(route=route, markers=_MODEL_ROUTING_LLM_HANDLER_ROUTE_MARKERS):
|
||||
# The handler routes from ``?model=`` / ``{model}`` ahead of the body, so
|
||||
# every model it can end up dispatching to must pass the allowlist. The
|
||||
# ``x-litellm-model`` header is deliberately not read here: these handlers
|
||||
# do not route from it.
|
||||
_append_model_candidates(candidates, _get_case_insensitive_mapping_value(request_query_params, "model"))
|
||||
_append_model_candidates(candidates, _llm_handler_path_model(route))
|
||||
# URL-bound model on the LLM handler routes: request processing prefers it
|
||||
# over the body model, so it must pass the allowlist as well. The
|
||||
# ``x-litellm-model`` header is deliberately not read: these handlers do
|
||||
# not route from it.
|
||||
_append_model_candidates(
|
||||
candidates,
|
||||
_llm_handler_url_model(
|
||||
route=route,
|
||||
route_template=route_template,
|
||||
request_path_params=request_path_params,
|
||||
request_query_params=request_query_params,
|
||||
),
|
||||
)
|
||||
|
||||
if uses_model_routing_sources:
|
||||
if uses_header_or_query_model_sources:
|
||||
|
|
@ -2037,6 +2105,8 @@ def get_model_from_request(
|
|||
request_query_params=request_query_params,
|
||||
llm_router=llm_router,
|
||||
team_id=team_id,
|
||||
route_template=get_request_route_template(request) if request is not None else None,
|
||||
request_path_params=_safe_get_request_path_params(request),
|
||||
)
|
||||
model = _format_model_candidates(candidates)
|
||||
|
||||
|
|
|
|||
|
|
@ -3919,11 +3919,22 @@ class TestIsRequestBodySafeBlocksAwsIdentitySelectors:
|
|||
is True
|
||||
)
|
||||
|
||||
# --- query-string / path model must be allowlisted on LLM handler routes ------------
|
||||
# The chat-completions, completions and embeddings handlers route from a ``model``
|
||||
# query or path parameter in preference to the body model. Only the body used to
|
||||
# be validated, so a key scoped to one model could be served another by putting
|
||||
# it in ``?model=`` (BerriAI/litellm#41810).
|
||||
# --- URL-bound model must be allowlisted on the LLM handler routes -------------------
|
||||
# chat-completions, completions and embeddings bind ``model`` from the URL
|
||||
# (``?model=`` on the plain routes, ``{model:path}`` on the deployment-style
|
||||
# routes) and route from it ahead of the body model. Only the body used to be
|
||||
# validated, so a key scoped to one model could be served another
|
||||
# (BerriAI/litellm#41810).
|
||||
|
||||
|
||||
def _matched_request(template: str, path_params: dict | None = None) -> Request:
|
||||
"""Request as Starlette leaves it after routing: ``scope["route"]`` carries the
|
||||
matched template and ``path_params`` the resolved segments."""
|
||||
|
||||
class _Route:
|
||||
path = template
|
||||
|
||||
return Request(scope={"type": "http", "headers": [], "route": _Route(), "path_params": path_params or {}})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -3931,6 +3942,7 @@ class TestIsRequestBodySafeBlocksAwsIdentitySelectors:
|
|||
[
|
||||
"/v1/chat/completions",
|
||||
"/chat/completions",
|
||||
"/queue/chat/completions",
|
||||
"/v1/completions",
|
||||
"/completions",
|
||||
"/v1/embeddings",
|
||||
|
|
@ -3944,6 +3956,16 @@ def test_get_model_from_request_includes_query_model_on_llm_handler_routes(route
|
|||
request_data={"model": "allowed-model"},
|
||||
route=route,
|
||||
request_query_params={"model": "denied-model"},
|
||||
request=_matched_request(route),
|
||||
) == ["allowed-model", "denied-model"]
|
||||
|
||||
|
||||
def test_get_model_from_request_query_model_without_request_object():
|
||||
"""Callers with no Request (budget reservation) classify the literal path."""
|
||||
assert get_model_from_request(
|
||||
request_data={"model": "allowed-model"},
|
||||
route="/v1/chat/completions",
|
||||
request_query_params={"model": "denied-model"},
|
||||
) == ["allowed-model", "denied-model"]
|
||||
|
||||
|
||||
|
|
@ -3953,6 +3975,7 @@ def test_get_model_from_request_query_model_alone_on_chat_completions():
|
|||
request_data={"messages": []},
|
||||
route="/v1/chat/completions",
|
||||
request_query_params={"model": "denied-model"},
|
||||
request=_matched_request("/v1/chat/completions"),
|
||||
)
|
||||
== "denied-model"
|
||||
)
|
||||
|
|
@ -3964,37 +3987,110 @@ def test_get_model_from_request_query_model_equal_to_body_dedupes():
|
|||
request_data={"model": "allowed-model"},
|
||||
route="/v1/chat/completions",
|
||||
request_query_params={"model": "allowed-model"},
|
||||
request=_matched_request("/v1/chat/completions"),
|
||||
)
|
||||
== "allowed-model"
|
||||
)
|
||||
|
||||
|
||||
def test_get_model_from_request_query_model_lookup_is_exact_case():
|
||||
"""FastAPI binds ``model`` case-sensitively: ``?MODEL=`` is not bound by the
|
||||
handler, so it must not become a candidate either (it would only false-403)."""
|
||||
assert (
|
||||
get_model_from_request(
|
||||
request_data={"model": "allowed-model"},
|
||||
route="/v1/chat/completions",
|
||||
request_query_params={"MODEL": "denied-model"},
|
||||
request=_matched_request("/v1/chat/completions"),
|
||||
)
|
||||
== "allowed-model"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route, expected",
|
||||
"template, path, path_model",
|
||||
[
|
||||
("/openai/deployments/denied-model/chat/completions", ["allowed-model", "denied-model"]),
|
||||
("/openai/deployments/denied-model/completions", ["allowed-model", "denied-model"]),
|
||||
("/openai/deployments/denied-model/embeddings", ["allowed-model", "denied-model"]),
|
||||
("/engines/denied-model/chat/completions", ["allowed-model", "denied-model"]),
|
||||
("/openai/deployments/{model:path}/chat/completions", "/openai/deployments/denied-model/chat/completions", "denied-model"),
|
||||
("/openai/deployments/{model:path}/completions", "/openai/deployments/denied-model/completions", "denied-model"),
|
||||
("/openai/deployments/{model:path}/embeddings", "/openai/deployments/denied-model/embeddings", "denied-model"),
|
||||
("/engines/{model:path}/chat/completions", "/engines/denied-model/chat/completions", "denied-model"),
|
||||
("/engines/{model:path}/completions", "/engines/denied-model/completions", "denied-model"),
|
||||
("/engines/{model:path}/embeddings", "/engines/denied-model/embeddings", "denied-model"),
|
||||
# {model:path} may contain slashes
|
||||
("/openai/deployments/org/denied-model/chat/completions", ["allowed-model", "org/denied-model"]),
|
||||
("/openai/deployments/{model:path}/chat/completions", "/openai/deployments/org/denied-model/chat/completions", "org/denied-model"),
|
||||
],
|
||||
)
|
||||
def test_get_model_from_request_includes_deployment_path_model(route, expected):
|
||||
"""The ``{model}`` path segment is bound to the handler and routed from even when
|
||||
def test_get_model_from_request_includes_deployment_path_model(template, path, path_model):
|
||||
"""The ``{model:path}`` segment is bound to the handler and routed from even when
|
||||
the body carries a different model, so it is validated alongside the body."""
|
||||
assert get_model_from_request(request_data={"model": "allowed-model"}, route=route) == expected
|
||||
assert get_model_from_request(
|
||||
request_data={"model": "allowed-model"},
|
||||
route=path,
|
||||
request=_matched_request(template, {"model": path_model}),
|
||||
) == ["allowed-model", path_model]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ["/v1/messages", "/v1/responses"])
|
||||
def test_get_model_from_request_ignores_query_model_on_routes_that_do_not_route_from_it(route):
|
||||
"""/v1/messages and /v1/responses serve the body model regardless of ``?model=``,
|
||||
so the query value is not a candidate there (unchanged behaviour)."""
|
||||
def test_get_model_from_request_deployment_path_model_without_request_object():
|
||||
"""No Request: the literal path is parsed, slashes in the model included."""
|
||||
assert get_model_from_request(
|
||||
request_data={"model": "allowed-model"},
|
||||
route="/openai/deployments/org/denied-model/chat/completions",
|
||||
) == ["allowed-model", "org/denied-model"]
|
||||
|
||||
|
||||
def test_get_model_from_request_deployment_route_ignores_query_model():
|
||||
"""On a ``{model:path}`` route FastAPI binds the path value and ignores a
|
||||
same-named query parameter, so auth validates exactly what will be routed."""
|
||||
assert (
|
||||
get_model_from_request(
|
||||
request_data={"model": "allowed-model"},
|
||||
route="/openai/deployments/allowed-model/chat/completions",
|
||||
request_query_params={"model": "denied-model"},
|
||||
request=_matched_request("/openai/deployments/{model:path}/chat/completions", {"model": "allowed-model"}),
|
||||
)
|
||||
== "allowed-model"
|
||||
)
|
||||
|
||||
|
||||
def test_get_model_from_request_prefers_resolved_path_params_over_literal_path():
|
||||
"""The value FastAPI resolved wins over re-parsing the path (encoding, slashes)."""
|
||||
assert get_model_from_request(
|
||||
request_data={"model": "allowed-model"},
|
||||
route="/openai/deployments/a%2Fb/chat/completions",
|
||||
request=_matched_request("/openai/deployments/{model:path}/chat/completions", {"model": "a/b"}),
|
||||
) == ["allowed-model", "a/b"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route, template",
|
||||
[
|
||||
("/v1/messages", "/v1/messages"),
|
||||
("/v1/responses", "/v1/responses"),
|
||||
("/cursor/chat/completions", "/cursor/chat/completions"),
|
||||
],
|
||||
)
|
||||
def test_get_model_from_request_ignores_query_model_on_routes_that_do_not_bind_it(route, template):
|
||||
"""Routes whose handler does not bind ``?model=`` (including ones that merely
|
||||
contain ``/chat/completions`` in the path) keep serving the body model, so the
|
||||
query value is not a candidate there."""
|
||||
assert (
|
||||
get_model_from_request(
|
||||
request_data={"model": "allowed-model"},
|
||||
route=route,
|
||||
request_query_params={"model": "denied-model"},
|
||||
request=_matched_request(template),
|
||||
)
|
||||
== "allowed-model"
|
||||
)
|
||||
|
||||
|
||||
def test_get_model_from_request_azure_relay_path_is_not_treated_as_deployment_route():
|
||||
"""``/azure/openai/deployments/...`` is the Azure relay with its own resolver;
|
||||
without a Request it must not be parsed as a plain deployment route either."""
|
||||
assert (
|
||||
get_model_from_request(
|
||||
request_data={"model": "allowed-model"},
|
||||
route="/azure/openai/deployments/denied-model/chat/completions",
|
||||
)
|
||||
== "allowed-model"
|
||||
)
|
||||
|
|
@ -4008,6 +4104,7 @@ def test_get_model_from_request_ignores_routing_header_on_chat_completions():
|
|||
request_data={"model": "allowed-model"},
|
||||
route="/v1/chat/completions",
|
||||
request_headers={"x-litellm-model": "denied-model"},
|
||||
request=_matched_request("/v1/chat/completions"),
|
||||
)
|
||||
== "allowed-model"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue