From 4a68abfd49eea3b7d4fdfeaadd55598369848ef2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:26:23 +0000 Subject: [PATCH] fix(proxy): route container create and list through model_list deployments Container create and list requests had no container ID to decode, so the router called the provider handler directly and the OpenAI transformation fell back to the global OPENAI_API_KEY. Proxies configured only with model_list credentials sent Authorization: Bearer None. Route through _ageneric_api_call_with_fallbacks when the caller passes a model, expose the list endpoint's model query param to the router, and encode the managed container ID on the async create path so follow-up calls route to the same deployment. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/containers/main.py | 25 +++++--- .../proxy/container_endpoints/endpoints.py | 2 +- litellm/router.py | 11 +++- .../test_router_endpoints.py | 59 +++++++++++++++++++ .../containers/test_container_api.py | 36 +++++++++++ 5 files changed, 122 insertions(+), 11 deletions(-) diff --git a/litellm/containers/main.py b/litellm/containers/main.py index 97ca11872c1..90d59af009f 100644 --- a/litellm/containers/main.py +++ b/litellm/containers/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars import json -from collections.abc import Coroutine, Mapping +from collections.abc import Callable, Coroutine, Mapping from functools import partial from typing import Final, Literal, overload @@ -47,6 +47,13 @@ __all__ = [ ##### Container Create ####################### +async def _encode_created_container_id( + pending: Coroutine[object, object, ContainerObject], + encode: Callable[[ContainerObject], ContainerObject], +) -> ContainerObject: + return encode(await pending) + + @client async def acreate_container( name: str, @@ -256,16 +263,16 @@ def create_container( _is_async=_is_async, ) - # Encode container_id with provider/model metadata for routing + encode: Final = partial( + ContainerRequestUtils.encode_container_id_in_response, + custom_llm_provider=custom_llm_provider, + litellm_metadata=kwargs.get("litellm_metadata"), + extra_body=extra_body, + ) if isinstance(container_obj, ContainerObject): - container_obj = ContainerRequestUtils.encode_container_id_in_response( - response_obj=container_obj, - custom_llm_provider=custom_llm_provider, - litellm_metadata=kwargs.get("litellm_metadata"), - extra_body=extra_body, - ) + return encode(container_obj) - return container_obj + return _encode_created_container_id(pending=container_obj, encode=encode) except Exception as e: raise litellm.exception_type( diff --git a/litellm/proxy/container_endpoints/endpoints.py b/litellm/proxy/container_endpoints/endpoints.py index 4a088140725..eaa3db336a9 100644 --- a/litellm/proxy/container_endpoints/endpoints.py +++ b/litellm/proxy/container_endpoints/endpoints.py @@ -208,7 +208,7 @@ async def list_containers( # Read query parameters query_params: Final = dict(request.query_params) - data: Final[dict[str, Any]] = {"query_params": query_params} + data: Final[dict[str, Any]] = {"query_params": query_params, "model": query_params.get("model")} # Extract custom_llm_provider using priority chain custom_llm_provider: Final = ( diff --git a/litellm/router.py b/litellm/router.py index 471a1116f44..52e858a0b6c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6589,7 +6589,9 @@ class Router: metadata. When present, decode the ID, replace ``container_id`` with the upstream value, and route through ``_ageneric_api_call_with_fallbacks`` so deployment credentials (e.g. regional ``api_base`` for Azure) match - :meth:`_init_responses_api_endpoints`. Otherwise call the handler directly. + :meth:`_init_responses_api_endpoints`. Create/list calls carry no container ID, so + they route through the deployment named by ``model`` when the caller passes one. + Otherwise call the handler directly with global provider credentials. """ if custom_llm_provider and "custom_llm_provider" not in kwargs: kwargs["custom_llm_provider"] = custom_llm_provider @@ -6621,6 +6623,13 @@ class Router: **kwargs, ) + requested_model: Final = kwargs.get("model") + if isinstance(requested_model, str) and requested_model.strip(): + return await self._ageneric_api_call_with_fallbacks( + original_function=original_function, + **kwargs, + ) + return await original_function(**kwargs) async def _init_responses_api_endpoints( diff --git a/tests/router_unit_tests/test_router_endpoints.py b/tests/router_unit_tests/test_router_endpoints.py index d37af5b456a..6fcbd77e054 100644 --- a/tests/router_unit_tests/test_router_endpoints.py +++ b/tests/router_unit_tests/test_router_endpoints.py @@ -1324,6 +1324,65 @@ async def test_init_containers_api_endpoints_managed_id_without_model_id_applies assert call_kw["custom_llm_provider"] == "azure" +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_with_model_uses_deployment_credentials(monkeypatch): + """ + ``POST /v1/containers`` carries no container ID, so a ``model`` in the request + body is the only way to pick a deployment. The upstream call must receive that + deployment's ``api_key``/``api_base`` instead of falling back to the global + ``OPENAI_API_KEY`` (which may be unset on the proxy). + """ + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + router = Router( + model_list=[ + { + "model_name": "gpt-5.4", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_key": "sk-model-list-key", + "api_base": "https://custom.openai.example/v1", + }, + } + ] + ) + mock_original_function = AsyncMock(return_value={"id": "cntr_test", "name": "Test Container"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model="gpt-5.4", + ) + + mock_original_function.assert_called_once() + call_kw = mock_original_function.call_args.kwargs + assert call_kw["api_key"] == "sk-model-list-key" + assert call_kw["api_base"] == "https://custom.openai.example/v1" + assert call_kw["model"] == "openai/gpt-5.4" + assert call_kw["name"] == "Test Container" + + +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_without_model_calls_directly(): + """ + Without ``model`` (or with ``model=None`` as the proxy forwards it), create/list + must keep calling the handler directly with global provider credentials. + """ + router = Router(model_list=[]) + router._ageneric_api_call_with_fallbacks = AsyncMock() + mock_original_function = AsyncMock(return_value={"id": "cntr_test"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model=None, + ) + + router._ageneric_api_call_with_fallbacks.assert_not_called() + mock_original_function.assert_called_once_with(custom_llm_provider="openai", name="Test Container", model=None) + + def test_router_model_group_encrypted_content_affinity_callback_registration(): from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, diff --git a/tests/test_litellm/containers/test_container_api.py b/tests/test_litellm/containers/test_container_api.py index 885c4cd294a..8f1d87f9603 100644 --- a/tests/test_litellm/containers/test_container_api.py +++ b/tests/test_litellm/containers/test_container_api.py @@ -152,6 +152,42 @@ class TestContainerAPI: assert response.id == "cntr_async_123" assert response.name == "Async Test Container" + @pytest.mark.asyncio + async def test_acreate_container_encodes_router_model_id(self): + """ + The async handler returns a coroutine, so the managed-ID encoding must run + after it resolves. Otherwise follow-up calls (retrieve/delete/files) lose the + deployment and fall back to global provider credentials. + """ + upstream_response = ContainerObject( + id="cntr_upstream_123", + object="container", + created_at=1747857508, + status="running", + expires_after={"anchor": "last_active_at", "minutes": 20}, + last_active_at=1747857508, + name="Routed Container", + ) + + async def _resolve_upstream(): + return upstream_response + + with patch.object( # test-quality-ok: create_container does not forward a client, so the handler is the only seam + base_llm_http_handler, + "container_create_handler", + side_effect=lambda **kwargs: _resolve_upstream() if kwargs["_is_async"] else upstream_response, + ): + response = await acreate_container( + name="Routed Container", + custom_llm_provider="openai", + litellm_metadata={"model_info": {"id": "deployment-abc"}}, + ) + + decoded = ResponsesAPIRequestUtils._decode_container_id(response.id) + assert decoded["model_id"] == "deployment-abc" + assert decoded["custom_llm_provider"] == "openai" + assert decoded["response_id"] == "cntr_upstream_123" + @pytest.mark.asyncio async def test_alist_containers_basic(self): """Test basic async container listing functionality."""