diff --git a/litellm/llms/azure/decisions/transformation.py b/litellm/llms/azure/decisions/transformation.py index bdbd784185f..87b6cc5fc66 100644 --- a/litellm/llms/azure/decisions/transformation.py +++ b/litellm/llms/azure/decisions/transformation.py @@ -5,6 +5,7 @@ from typing import Final from pydantic import ValidationError import litellm +from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import ( BaseAzureLLM, cached_azure_ad_token_refresh_provider, @@ -74,5 +75,8 @@ class AzureDecisionsConfig(OpenAIDecisionsConfig): ) -> Mapping[str, str]: return await asyncio.to_thread(self.validate_environment, headers, model, api_key, litellm_params) + def canonical_model(self, model: str) -> str: + return model.removeprefix(AzureOpenAIGPT5Config.GPT5_SERIES_ROUTE) + def get_complete_url(self, api_base: str, model: str) -> str: return BaseAzureLLM.get_base_azure_url(api_base, None, route="/openai/v1/decisions") diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 680b0e6ede4..92b01d5dec0 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -290,7 +290,16 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( LazyFeature( name="decisions", module_path="litellm.proxy.decisions_endpoints.endpoints", - path_prefixes=("/v1/decisions", "/decisions", "/v1/systemone", "/systemone", "/v1/extraction", "/extraction"), + path_prefixes=( + "/v1/decisions", + "/decisions", + "/openai/v1/decisions", + "/openai/decisions", + "/v1/systemone", + "/systemone", + "/v1/extraction", + "/extraction", + ), ), LazyFeature( name="skill_hub", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e53b5d4ad52..241c0b34662 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -386,6 +386,8 @@ class LiteLLMRoutes(enum.Enum): "/v1/search/{search_tool_name}", "/decisions", "/v1/decisions", + "/openai/v1/decisions", + "/openai/decisions", "/systemone", "/v1/systemone", "/extraction", diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index 57e99edecba..39b22fa9b9b 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -47,6 +47,8 @@ _MANAGED_MODEL_ROUTES: Final = frozenset( ) | frozenset( ( "/openai/v1/responses", + "/openai/v1/decisions", + "/openai/decisions", "/v2/rerank", "/claude_code_gateway/v1/messages", "/claude_code_gateway/v1/messages/count_tokens", diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index a8684dd9757..77b9c58b75d 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -10,6 +10,7 @@ from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import YieldsToConfiguredPassThroughRoute from litellm.types.decisions import ( DecisionsRequest, DecisionsRequestBody, @@ -20,7 +21,7 @@ from litellm.types.decisions import ( ) from litellm.types.extraction import ExtractionRequest, ExtractionRequestBody, ExtractionResponse -router: Final = APIRouter() +router: Final = APIRouter(route_class=YieldsToConfiguredPassThroughRoute) _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) _OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter( @@ -248,6 +249,20 @@ async def systemone( ) +@router.post( + "/openai/v1/decisions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], + include_in_schema=False, +) +@router.post( + "/openai/decisions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], + include_in_schema=False, +) @router.post( "/v1/decisions", dependencies=[Depends(user_api_key_auth)], diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3297a298af4..77efb70c0c1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -22463,6 +22463,7 @@ app.include_router(tracing_endpoints.router) app.include_router(image_router) app.include_router(fine_tuning_router) app.include_router(credential_router) +reserve_lazy_slot(app, "decisions") app.include_router(openai_passthrough_router) app.include_router(batches_router) app.include_router(openai_files_router) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3ddeb118c33..ec51f1c36fa 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -891,6 +891,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/search": [CallTypes.asearch, CallTypes.search], "/decisions": [CallTypes.adecisions, CallTypes.decisions], "/v1/decisions": [CallTypes.adecisions, CallTypes.decisions], + "/openai/v1/decisions": [CallTypes.adecisions, CallTypes.decisions], + "/openai/decisions": [CallTypes.adecisions, CallTypes.decisions], "/systemone": [CallTypes.asystemone, CallTypes.systemone], "/v1/systemone": [CallTypes.asystemone, CallTypes.systemone], "/extraction": [CallTypes.aextraction, CallTypes.extraction], diff --git a/tests/unit/llms/azure/decisions/test_transformation.py b/tests/unit/llms/azure/decisions/test_transformation.py index 3f5e97ee000..2afc55a7298 100644 --- a/tests/unit/llms/azure/decisions/test_transformation.py +++ b/tests/unit/llms/azure/decisions/test_transformation.py @@ -53,9 +53,10 @@ async def call_decisions( systemone: bool, credentials: Mapping[str, object], api_base: str = _BASE, + model: str = "azure/luna-deployment", ) -> DecisionsResponse | OpenAIDecisionResponse: params = { - "model": "azure/luna-deployment", + "model": model, "api_base": api_base, **credentials, } @@ -220,6 +221,15 @@ async def test_azure_credential_lookup_does_not_block_the_event_loop( await call_decisions(True, False, {"azure_ad_token_provider": provider}) +@pytest.mark.parametrize("systemone", (False, True)) +async def test_gpt5_series_selector_is_not_sent_as_part_of_the_deployment_name( + respx_mock: respx.MockRouter, systemone: bool +) -> None: + route = respx_mock.post(_URL).respond(json=_RESPONSE) + await call_decisions(True, systemone, {"api_key": "synthetic-key"}, model="azure/gpt5_series/luna-deployment") + assert json.loads(route.calls.last.request.content)["model"] == "luna-deployment" + + def test_azure_decisions_requires_azure_endpoint( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index c67d1012475..df35f41903e 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -6,7 +6,7 @@ import json import random import sys from collections.abc import AsyncGenerator, Callable, Iterator, Mapping -from contextlib import asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager from pathlib import Path from typing import Final, Literal @@ -22,11 +22,15 @@ import litellm from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing -from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features +from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.decisions_endpoints.endpoints import decisions, router, systemone -from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, +) from litellm.proxy.proxy_server import ( app, cleanup_router_config_variables, @@ -765,6 +769,84 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_ assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) +_AZURE_SDK_DECISIONS_PATHS: Final = ( + "/openai/v1/decisions", + "/openai/v1/decisions?api-version=preview", + "/openai/decisions?api-version=2025-04-01-preview", +) + + +def _route_to_an_azure_deployment(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> respx.Route: + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "decider", + "litellm_params": { + "model": "azure/gpt5_series/my-decisions-deployment", + "api_base": "https://my-resource.openai.azure.com", + "api_key": "azure-deployment-key", + }, + } + ] + ), + ) + return respx_mock.post("https://my-resource.openai.azure.com/openai/v1/decisions").respond( + json={ + "model": "gpt-6-luna", + "answers": _OPENAI_FORMAT_ANSWERS, + "usage": { + "input_tokens": _INPUT_TOKENS, + "output_tokens": _OUTPUT_TOKENS, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + }, + } + ) + + +@pytest.mark.parametrize("path", _AZURE_SDK_DECISIONS_PATHS) +def test_azure_sdk_decisions_paths_reach_the_azure_deployment( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + path: str, +) -> None: + upstream: Final = _route_to_an_azure_deployment(monkeypatch, respx_mock) + + response: Final = client.post(path, json=_OPENAI_FORMAT_REQUEST, headers={"api-key": "caller-key"}) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == _OPENAI_FORMAT_ANSWERS + assert upstream.call_count == 1 + sent: Final = upstream.calls[0].request + assert sent.headers["api-key"] == "azure-deployment-key" + assert json.loads(sent.content)["model"] == "my-decisions-deployment" + luna_decisions_cost: Final = apply_decisions_pricing(litellm.model_cost["azure/gpt-6-luna"]) + expected_cost: Final = _INPUT_TOKENS * float(luna_decisions_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( + luna_decisions_cost["output_cost_per_token"] + ) + assert expected_cost > 0 + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) + + +@pytest.mark.parametrize("path", _AZURE_SDK_DECISIONS_PATHS) +def test_azure_sdk_decisions_paths_require_a_virtual_key( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + path: str, +) -> None: + upstream: Final = _route_to_an_azure_deployment(monkeypatch, respx_mock) + monkeypatch.setattr(litellm.proxy.proxy_server, "master_key", MASTER_KEY) + + response: Final = client.post(path, json=_OPENAI_FORMAT_REQUEST) + + assert response.status_code == 401, response.text + assert not upstream.called + + _CLEF_OMNI_MEDIA: Final[Mapping[str, object]] = { "audio": ["data:audio/wav;base64,YQ==", {"content_type": "audio/webm;codecs=opus", "base64": "Yg=="}], "videos": ["data:video/mp4;base64,Yw=="], @@ -1244,20 +1326,58 @@ def _serving_endpoint(bare: FastAPI, path: str) -> object: ) -def test_a_config_pass_through_at_v1_decisions_keeps_its_route_and_the_native_api_serves_decisions( +@pytest.fixture +def config_pass_through_upstream(monkeypatch: pytest.MonkeyPatch) -> Iterator[respx.Route]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + saved_registry: Final = dict(_registered_pass_through_routes) + try: + with respx.mock(assert_all_called=False) as upstream: + yield upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + + +def _add_config_pass_through(app_: FastAPI, path: str) -> None: + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app_, + path=path, + target="https://operator.example/decide", + custom_headers={}, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="decisions-operator", + methods=["POST"], + ) + + +def _app_with_the_proxy_decisions_slot( monkeypatch: pytest.MonkeyPatch, + lifespan: Callable[[FastAPI], AbstractAsyncContextManager[None]] | None = None, +) -> FastAPI: + bare: Final = FastAPI(lifespan=lifespan) + reserve_lazy_slot(bare, "decisions", features=(_decisions_feature(),)) + attach_lazy_features(bare, (_decisions_feature(),)) + monkeypatch.setitem(bare.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + return bare + + +@pytest.mark.parametrize("path", ("/v1/decisions", "/openai/v1/decisions")) +def test_a_config_pass_through_at_a_decisions_path_keeps_its_route_and_the_native_api_serves_decisions( + monkeypatch: pytest.MonkeyPatch, + config_pass_through_upstream: respx.Route, + path: str, ) -> None: monkeypatch.delenv("LITELLM_DISABLE_LAZY_ROUTES", raising=False) - async def pass_through() -> dict[str, str]: - return {"served_by": "pass-through"} - - bare: Final = FastAPI() - attach_lazy_features(bare, (_decisions_feature(),)) - SafeRouteAdder.add_api_route_if_not_exists(bare, "/v1/decisions", pass_through, ["POST"]) + bare: Final = _app_with_the_proxy_decisions_slot(monkeypatch) + _add_config_pass_through(bare, path) with TestClient(bare) as client: - assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} - assert _serving_endpoint(bare, "/v1/decisions") is pass_through + assert client.post(path, json={"model": "gpt-6-luna"}).json() == {"operator": True} + assert config_pass_through_upstream.call_count == 1 assert _serving_endpoint(bare, "/decisions") is decisions assert _serving_endpoint(bare, "/v1/systemone") is systemone assert _serving_endpoint(bare, "/systemone") is systemone @@ -1265,24 +1385,19 @@ def test_a_config_pass_through_at_v1_decisions_keeps_its_route_and_the_native_ap def test_with_lazy_routes_disabled_a_config_pass_through_at_v1_decisions_still_wins( monkeypatch: pytest.MonkeyPatch, + config_pass_through_upstream: respx.Route, ) -> None: monkeypatch.setenv("LITELLM_DISABLE_LAZY_ROUTES", "true") - async def pass_through() -> dict[str, str]: - return {"served_by": "pass-through"} - @asynccontextmanager async def loads_the_config(app_: FastAPI) -> AsyncGenerator[None]: - assert SafeRouteAdder.add_api_route_if_not_exists(app_, "/v1/decisions", pass_through, ["POST"]), ( - "the native route registered at startup must not block the config pass-through" - ) + _add_config_pass_through(app_, "/v1/decisions") yield - bare: Final = FastAPI(lifespan=loads_the_config) - attach_lazy_features(bare, (_decisions_feature(),)) + bare: Final = _app_with_the_proxy_decisions_slot(monkeypatch, loads_the_config) with TestClient(bare) as client: - assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} - assert _serving_endpoint(bare, "/v1/decisions") is pass_through + assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"operator": True} + assert config_pass_through_upstream.call_count == 1 assert _serving_endpoint(bare, "/decisions") is decisions