diff --git a/litellm/constants.py b/litellm/constants.py index ca2be8af5fe..d84effde139 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -318,6 +318,7 @@ BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_real BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed" BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure" CLIENT_REQUESTED_MODEL_SCOPE_KEY: Final = "litellm.client_requested_model" +MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolved" REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged" REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged" diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index a02661db9ec..ac0f3696680 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -32,6 +32,7 @@ from litellm.constants import ( INVALID_VIRTUAL_KEY_ERROR_MESSAGE, LITELLM_PROXY_BUDGET_NAME, LITELLM_PROXY_MASTER_KEY_ALIAS, + MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY, ) from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.integrations.otel.runtime import phase_span, seed_request_identity @@ -104,6 +105,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _safe_set_request_parsed_body, populate_request_with_path_params, read_raw_json_body, + rewrite_request_model, ) from litellm.proxy.common_utils.model_listing_utils import claude_code_requested_group from litellm.proxy.common_utils.realtime_utils import _realtime_request_body @@ -237,22 +239,7 @@ async def _normalize_claude_model( request.scope[_CLAUDE_MODEL_NORMALIZED] = True if source is None: return - _rewrite_request_model(request_data, request, source) - - -def _rewrite_request_model( - request_data: dict, # mutable-ok: the request body is rewritten in place for every downstream reader - request: Request | None, - model: str, -) -> None: - request_data["model"] = model - _safe_set_request_parsed_body(request=request, parsed_body=request_data) - if request is not None: - request._json = request_data - request._body = orjson.dumps(request_data) - - -_MODEL_GROUP_ALIAS_RESOLVED: Final = "litellm.model_group_alias_resolved" + rewrite_request_model(request_data, request, source) async def _resolve_router_settings_model_group_alias( @@ -264,13 +251,16 @@ async def _resolve_router_settings_model_group_alias( """Rewrite the requested model through the key's or team's ``router_settings.model_group_alias`` before the allowlist checks, so they authorize the model group the request is routed to. """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers from litellm.proxy.proxy_server import llm_router, prisma_client, proxy_config, proxy_logging_obj if request is None or llm_router is None or not RouteChecks.is_llm_api_route(route=route): return - if request.scope.get(_MODEL_GROUP_ALIAS_RESOLVED) is True: + if request.scope.get(MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY) is True: + return + request.scope[MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY] = True + if InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=route): return - request.scope[_MODEL_GROUP_ALIAS_RESOLVED] = True requested: Final = request_data.get("model") if not isinstance(requested, str) or await read_raw_json_body(request=request) is None: return @@ -284,7 +274,7 @@ async def _resolve_router_settings_model_group_alias( return verbose_proxy_logger.debug("router_settings.model_group_alias resolved %s -> %s before auth", requested, target) request.scope.setdefault(CLIENT_REQUESTED_MODEL_SCOPE_KEY, requested) - _rewrite_request_model(request_data, request, target) + rewrite_request_model(request_data, request, target) def _get_model_names_for_budget_checks( diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index ec2e05541cb..a1dcc6e8ece 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -266,6 +266,24 @@ def _safe_set_request_parsed_body( verbose_proxy_logger.debug("Unexpected error setting request parsed body - %s", e) +def rewrite_request_model( + request_data: dict, # mutable-ok: the request body is rewritten in place for every downstream reader + request: Request | None, + model: str, +) -> None: + """Point the auth-time payload, the parsed-body cache, ``request.json()`` and ``request.body()`` at ``model``. + The cache and raw body keep only the keys the client sent, not params auth merged into ``request_data``. + """ + request_data["model"] = model + if request is None: + return + cached_body: Final = _safe_get_request_parsed_body(request=request) + body: Final = {**cached_body, "model": model} if cached_body is not None else request_data + _safe_set_request_parsed_body(request=request, parsed_body=body) + request._json = body + request._body = orjson.dumps(body) + + def _safe_get_request_headers(request: Request | None) -> dict: """ [Non-Blocking] Safely get the request headers. diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index ded45f756be..f974e4d29d2 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -8209,6 +8209,44 @@ async def test_router_settings_model_group_alias_leaves_form_bodies_alone(monkey assert get_client_requested_model(request) is None +@pytest.mark.asyncio +async def test_router_settings_model_group_alias_rewrite_keeps_query_params_out_of_body(monkeypatch): + """LIT-3054: auth merges query params into its own copy of the body; the rewrite must not forward them.""" + from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body, populate_request_with_path_params + + router = _alias_router() + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + body = {"model": "AgentX-LLM", "messages": [{"role": "user", "content": "hi"}]} + request = _alias_request("/v1/chat/completions", body) + request.scope["query_string"] = b"api-version=2024-10-21&stream=true" + data = populate_request_with_path_params(request_data=await _read_request_body(request), request=request) + assert data["api-version"] == "2024-10-21" + token = _alias_token(monkeypatch, "key", {"AgentX-LLM": "claude-haiku"}, ["claude-haiku"]) + await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route="/v1/chat/completions", request=request, llm_model_list=router.model_list, llm_router=router) + downstream = await _read_request_body(request) + assert downstream == {**body, "model": "claude-haiku"} + assert json.loads(await request.body()) == downstream + assert await request.json() == downstream + + +@pytest.mark.asyncio +async def test_router_settings_model_group_alias_leaves_pass_through_bodies_alone(monkeypatch): + """LIT-3054: pass-through routes forward the body verbatim to the provider, so auth must not rewrite it.""" + from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access + + router = _alias_router() + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + data = {"model": "AgentX-LLM", "messages": [{"role": "user", "content": "hi"}]} + route = "/anthropic/v1/messages" + request = _alias_request(route, data) + token = _alias_token(monkeypatch, "key", {"AgentX-LLM": "claude-haiku"}, ["claude-haiku", "AgentX-LLM"]) + await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route=route, request=request, llm_model_list=router.model_list, llm_router=router) + assert data["model"] == "AgentX-LLM" + assert (await request.json())["model"] == "AgentX-LLM" + assert get_client_requested_model(request) is None + + @pytest.mark.asyncio @pytest.mark.parametrize("target, expect_denied", [("claude-haiku", False), ("claude-sonnet", True)]) async def test_router_settings_model_group_alias_authorizes_target_for_team(monkeypatch, target, expect_denied):