mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): run Ask AI provider model strings on the environment keys, as before this change
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
This commit is contained in:
parent
2279307a14
commit
c19614d7d1
2 changed files with 10 additions and 60 deletions
|
|
@ -8,7 +8,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable,
|
|||
from datetime import date
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Protocol, TypeAlias, cast, overload
|
||||
from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Protocol, cast, overload
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
|
||||
|
|
@ -471,23 +471,12 @@ def _resolve_model(model: str | None) -> str:
|
|||
return (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
|
||||
|
||||
|
||||
_RouterRoute: TypeAlias = Literal["model_group", "deployment"]
|
||||
|
||||
|
||||
def _router_route(
|
||||
router: "Router", model: str, caller_is_proxy_admin: bool, request_team_id: str | None
|
||||
) -> _RouterRoute | None:
|
||||
def _is_router_model(router: "Router", model: str, caller_is_proxy_admin: bool) -> bool:
|
||||
if router.is_recognized_model(model) or (caller_is_proxy_admin and model in router.team_public_model_names):
|
||||
return "model_group"
|
||||
return True
|
||||
if router.router_general_settings.pass_through_all_models:
|
||||
return None
|
||||
if router.default_deployment is not None or router.pattern_router.route(model) is not None: # pyright: ignore[reportUnknownMemberType] # PatternMatchDeployments.route returns untyped dicts
|
||||
return "model_group"
|
||||
deployments: Final = router._get_deployment_by_litellm_model(model=model) # pyright: ignore[reportPrivateUsage] # the lookup the router's specific_deployment path runs
|
||||
serves_model: Final = any(
|
||||
router.deployment_usable_by_team(deployment, request_team_id) for deployment in deployments
|
||||
)
|
||||
return "deployment" if serves_model else None
|
||||
return False
|
||||
return router.default_deployment is not None or router.pattern_router.route(model) is not None # pyright: ignore[reportUnknownMemberType] # PatternMatchDeployments.route returns untyped dicts
|
||||
|
||||
|
||||
class _TaggedMetadata(BaseModel):
|
||||
|
|
@ -610,7 +599,7 @@ async def _acompletion(
|
|||
router_settings: Mapping[str, object] | None = None,
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
from litellm.router_utils.common_utils import get_request_team_id, is_proxy_admin_request
|
||||
from litellm.router_utils.common_utils import is_proxy_admin_request
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives
|
||||
|
|
@ -619,13 +608,9 @@ async def _acompletion(
|
|||
caller_metadata: Final[Mapping[str, object]] = (
|
||||
_caller_metadata(user_api_key_dict) if user_api_key_dict is not None else MappingProxyType({})
|
||||
)
|
||||
router_request: Final = MappingProxyType({"metadata": caller_metadata})
|
||||
route: Final = (
|
||||
_router_route(llm_router, model, is_proxy_admin_request(router_request), get_request_team_id(router_request))
|
||||
if llm_router is not None
|
||||
else None
|
||||
)
|
||||
if llm_router is not None and route is not None:
|
||||
if llm_router is not None and _is_router_model(
|
||||
llm_router, model, is_proxy_admin_request(MappingProxyType({"metadata": caller_metadata}))
|
||||
):
|
||||
router_kwargs: Final = _per_request_router_kwargs(router_settings)
|
||||
if stream:
|
||||
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
|
||||
|
|
@ -637,7 +622,6 @@ async def _acompletion(
|
|||
drop_params=True,
|
||||
allowed_openai_params=list(USAGE_AI_REQUIRED_PARAMS),
|
||||
metadata=dict(caller_metadata),
|
||||
specific_deployment=route == "deployment",
|
||||
**router_kwargs,
|
||||
)
|
||||
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
|
||||
|
|
@ -649,7 +633,6 @@ async def _acompletion(
|
|||
drop_params=True,
|
||||
allowed_openai_params=list(USAGE_AI_REQUIRED_PARAMS),
|
||||
metadata=dict(caller_metadata),
|
||||
specific_deployment=route == "deployment",
|
||||
**router_kwargs,
|
||||
)
|
||||
return await litellm.acompletion(
|
||||
|
|
|
|||
|
|
@ -646,46 +646,13 @@ class TestUsageAiChatProviderRequest:
|
|||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deployment_name_is_called_with_that_deployment_credentials(
|
||||
async def test_a_deployment_model_string_runs_on_the_environment_key(
|
||||
self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
self._mock_openai(respx_mock)
|
||||
|
||||
await self._ask("openai/gpt-5.6", monkeypatch, self._router())
|
||||
|
||||
assert respx_mock.calls.last.request.headers["Authorization"] == "Bearer sk-gpt5-deployment"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deleted_deployment_name_is_called_with_the_environment_key(
|
||||
self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
self._mock_openai(respx_mock)
|
||||
router: Final = self._router()
|
||||
router.delete_deployment(id=router.get_model_ids(model_name="gpt-5-prod")[0])
|
||||
|
||||
await self._ask("openai/gpt-5.6", monkeypatch, router)
|
||||
|
||||
assert respx_mock.calls.last.request.headers["Authorization"] == "Bearer sk-env"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_owned_deployment_name_is_called_with_the_environment_key(
|
||||
self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.router import Router
|
||||
|
||||
self._mock_openai(respx_mock)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "team-gpt-5",
|
||||
"litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-team-deployment"},
|
||||
"model_info": {"team_id": "team-1"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
await self._ask("openai/gpt-5.6", monkeypatch, router)
|
||||
|
||||
assert respx_mock.calls.last.request.headers["Authorization"] == "Bearer sk-env"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue