fix(proxy): run Ask AI provider model strings on the environment keys, as before this change
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

This commit is contained in:
mateo-berri 2026-10-20 21:33:24 -07:00
parent 2279307a14
commit c19614d7d1
2 changed files with 10 additions and 60 deletions

View file

@ -8,7 +8,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable,
from datetime import date
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Protocol, TypeAlias, cast, overload
from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Protocol, cast, overload
from fastapi import HTTPException
from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
@ -471,23 +471,12 @@ def _resolve_model(model: str | None) -> str:
return (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
_RouterRoute: TypeAlias = Literal["model_group", "deployment"]
def _router_route(
router: "Router", model: str, caller_is_proxy_admin: bool, request_team_id: str | None
) -> _RouterRoute | None:
def _is_router_model(router: "Router", model: str, caller_is_proxy_admin: bool) -> bool:
if router.is_recognized_model(model) or (caller_is_proxy_admin and model in router.team_public_model_names):
return "model_group"
return True
if router.router_general_settings.pass_through_all_models:
return None
if router.default_deployment is not None or router.pattern_router.route(model) is not None: # pyright: ignore[reportUnknownMemberType] # PatternMatchDeployments.route returns untyped dicts
return "model_group"
deployments: Final = router._get_deployment_by_litellm_model(model=model) # pyright: ignore[reportPrivateUsage] # the lookup the router's specific_deployment path runs
serves_model: Final = any(
router.deployment_usable_by_team(deployment, request_team_id) for deployment in deployments
)
return "deployment" if serves_model else None
return False
return router.default_deployment is not None or router.pattern_router.route(model) is not None # pyright: ignore[reportUnknownMemberType] # PatternMatchDeployments.route returns untyped dicts
class _TaggedMetadata(BaseModel):
@ -610,7 +599,7 @@ async def _acompletion(
router_settings: Mapping[str, object] | None = None,
) -> ModelResponse | CustomStreamWrapper:
from litellm.proxy.proxy_server import llm_router
from litellm.router_utils.common_utils import get_request_team_id, is_proxy_admin_request
from litellm.router_utils.common_utils import is_proxy_admin_request
from litellm.types.llms.openai import AllMessageValues
completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives
@ -619,13 +608,9 @@ async def _acompletion(
caller_metadata: Final[Mapping[str, object]] = (
_caller_metadata(user_api_key_dict) if user_api_key_dict is not None else MappingProxyType({})
)
router_request: Final = MappingProxyType({"metadata": caller_metadata})
route: Final = (
_router_route(llm_router, model, is_proxy_admin_request(router_request), get_request_team_id(router_request))
if llm_router is not None
else None
)
if llm_router is not None and route is not None:
if llm_router is not None and _is_router_model(
llm_router, model, is_proxy_admin_request(MappingProxyType({"metadata": caller_metadata}))
):
router_kwargs: Final = _per_request_router_kwargs(router_settings)
if stream:
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
@ -637,7 +622,6 @@ async def _acompletion(
drop_params=True,
allowed_openai_params=list(USAGE_AI_REQUIRED_PARAMS),
metadata=dict(caller_metadata),
specific_deployment=route == "deployment",
**router_kwargs,
)
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
@ -649,7 +633,6 @@ async def _acompletion(
drop_params=True,
allowed_openai_params=list(USAGE_AI_REQUIRED_PARAMS),
metadata=dict(caller_metadata),
specific_deployment=route == "deployment",
**router_kwargs,
)
return await litellm.acompletion(

View file

@ -646,46 +646,13 @@ class TestUsageAiChatProviderRequest:
]
@pytest.mark.asyncio
async def test_deployment_name_is_called_with_that_deployment_credentials(
async def test_a_deployment_model_string_runs_on_the_environment_key(
self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
self._mock_openai(respx_mock)
await self._ask("openai/gpt-5.6", monkeypatch, self._router())
assert respx_mock.calls.last.request.headers["Authorization"] == "Bearer sk-gpt5-deployment"
@pytest.mark.asyncio
async def test_deleted_deployment_name_is_called_with_the_environment_key(
self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
self._mock_openai(respx_mock)
router: Final = self._router()
router.delete_deployment(id=router.get_model_ids(model_name="gpt-5-prod")[0])
await self._ask("openai/gpt-5.6", monkeypatch, router)
assert respx_mock.calls.last.request.headers["Authorization"] == "Bearer sk-env"
@pytest.mark.asyncio
async def test_team_owned_deployment_name_is_called_with_the_environment_key(
self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.router import Router
self._mock_openai(respx_mock)
router: Final = Router(
model_list=[
{
"model_name": "team-gpt-5",
"litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-team-deployment"},
"model_info": {"team_id": "team-1"},
}
]
)
await self._ask("openai/gpt-5.6", monkeypatch, router)
assert respx_mock.calls.last.request.headers["Authorization"] == "Bearer sk-env"
@pytest.mark.asyncio