scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-05 08:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 137

fix(proxy): route container create and list through model_list deployments

Browse source 
Container create and list requests had no container ID to decode, so the
router called the provider handler directly and the OpenAI transformation
fell back to the global OPENAI_API_KEY. Proxies configured only with
model_list credentials sent Authorization: Bearer None. Route through
_ageneric_api_call_with_fallbacks when the caller passes a model, expose
the list endpoint's model query param to the router, and encode the
managed container ID on the async create path so follow-up calls route
to the same deployment.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-09-01 23:26:23 +00:00
parent fc1a5fd7f9
commit 4a68abfd49
5 changed files with 122 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
litellm/containers/main.py
View file

@ -1,7 +1,7 @@
import asyncio
import contextvars
import json
from collections.abc import Coroutine, Mapping
from collections.abc import Callable, Coroutine, Mapping
from functools import partial
from typing import Final, Literal, overload
@ -47,6 +47,13 @@ __all__ = [
##### Container Create #######################
async def _encode_created_container_id(
pending: Coroutine[object, object, ContainerObject],
encode: Callable[[ContainerObject], ContainerObject],
) -> ContainerObject:
return encode(await pending)
@client
async def acreate_container(
name: str,
@ -256,16 +263,16 @@ def create_container(
_is_async=_is_async,
)
# Encode container_id with provider/model metadata for routing
encode: Final = partial(
ContainerRequestUtils.encode_container_id_in_response,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata"),
extra_body=extra_body,
)
if isinstance(container_obj, ContainerObject):
container_obj = ContainerRequestUtils.encode_container_id_in_response(
response_obj=container_obj,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata"),
extra_body=extra_body,
)
return encode(container_obj)
return container_obj
return _encode_created_container_id(pending=container_obj, encode=encode)
except Exception as e:
raise litellm.exception_type(

2
litellm/proxy/container_endpoints/endpoints.py
View file

@ -208,7 +208,7 @@ async def list_containers(
# Read query parameters
query_params: Final = dict(request.query_params)
data: Final[dict[str, Any]] = {"query_params": query_params}
data: Final[dict[str, Any]] = {"query_params": query_params, "model": query_params.get("model")}
# Extract custom_llm_provider using priority chain
custom_llm_provider: Final = (

11
litellm/router.py
View file

@ -6589,7 +6589,9 @@ class Router:
metadata. When present, decode the ID, replace ``container_id`` with the
upstream value, and route through ``_ageneric_api_call_with_fallbacks`` so
deployment credentials (e.g. regional ``api_base`` for Azure) match
:meth:`_init_responses_api_endpoints`. Otherwise call the handler directly.
:meth:`_init_responses_api_endpoints`. Create/list calls carry no container ID, so
they route through the deployment named by ``model`` when the caller passes one.
Otherwise call the handler directly with global provider credentials.
"""
if custom_llm_provider and "custom_llm_provider" not in kwargs:
kwargs["custom_llm_provider"] = custom_llm_provider
@ -6621,6 +6623,13 @@ class Router:
**kwargs,
)
requested_model: Final = kwargs.get("model")
if isinstance(requested_model, str) and requested_model.strip():
return await self._ageneric_api_call_with_fallbacks(
original_function=original_function,
**kwargs,
)
return await original_function(**kwargs)
async def _init_responses_api_endpoints(

59
tests/router_unit_tests/test_router_endpoints.py
View file

@ -1324,6 +1324,65 @@ async def test_init_containers_api_endpoints_managed_id_without_model_id_applies
assert call_kw["custom_llm_provider"] == "azure"
@pytest.mark.asyncio
async def test_init_containers_api_endpoints_create_with_model_uses_deployment_credentials(monkeypatch):
"""
``POST /v1/containers`` carries no container ID, so a ``model`` in the request
body is the only way to pick a deployment. The upstream call must receive that
deployment's ``api_key``/``api_base`` instead of falling back to the global
``OPENAI_API_KEY`` (which may be unset on the proxy).
"""
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
router = Router(
model_list=[
{
"model_name": "gpt-5.4",
"litellm_params": {
"model": "openai/gpt-5.4",
"api_key": "sk-model-list-key",
"api_base": "https://custom.openai.example/v1",
},
}
]
)
mock_original_function = AsyncMock(return_value={"id": "cntr_test", "name": "Test Container"})
await router._init_containers_api_endpoints(
original_function=mock_original_function,
custom_llm_provider="openai",
name="Test Container",
model="gpt-5.4",
)
mock_original_function.assert_called_once()
call_kw = mock_original_function.call_args.kwargs
assert call_kw["api_key"] == "sk-model-list-key"
assert call_kw["api_base"] == "https://custom.openai.example/v1"
assert call_kw["model"] == "openai/gpt-5.4"
assert call_kw["name"] == "Test Container"
@pytest.mark.asyncio
async def test_init_containers_api_endpoints_create_without_model_calls_directly():
"""
Without ``model`` (or with ``model=None`` as the proxy forwards it), create/list
must keep calling the handler directly with global provider credentials.
"""
router = Router(model_list=[])
router._ageneric_api_call_with_fallbacks = AsyncMock()
mock_original_function = AsyncMock(return_value={"id": "cntr_test"})
await router._init_containers_api_endpoints(
original_function=mock_original_function,
custom_llm_provider="openai",
name="Test Container",
model=None,
)
router._ageneric_api_call_with_fallbacks.assert_not_called()
mock_original_function.assert_called_once_with(custom_llm_provider="openai", name="Test Container", model=None)
def test_router_model_group_encrypted_content_affinity_callback_registration():
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
DeploymentAffinityCheck,

36
tests/test_litellm/containers/test_container_api.py
View file

@ -152,6 +152,42 @@ class TestContainerAPI:
assert response.id == "cntr_async_123"
assert response.name == "Async Test Container"
@pytest.mark.asyncio
async def test_acreate_container_encodes_router_model_id(self):
"""
The async handler returns a coroutine, so the managed-ID encoding must run
after it resolves. Otherwise follow-up calls (retrieve/delete/files) lose the
deployment and fall back to global provider credentials.
"""
upstream_response = ContainerObject(
id="cntr_upstream_123",
object="container",
created_at=1747857508,
status="running",
expires_after={"anchor": "last_active_at", "minutes": 20},
last_active_at=1747857508,
name="Routed Container",
)
async def _resolve_upstream():
return upstream_response
with patch.object( # test-quality-ok: create_container does not forward a client, so the handler is the only seam
base_llm_http_handler,
"container_create_handler",
side_effect=lambda **kwargs: _resolve_upstream() if kwargs["_is_async"] else upstream_response,
):
response = await acreate_container(
name="Routed Container",
custom_llm_provider="openai",
litellm_metadata={"model_info": {"id": "deployment-abc"}},
)
decoded = ResponsesAPIRequestUtils._decode_container_id(response.id)
assert decoded["model_id"] == "deployment-abc"
assert decoded["custom_llm_provider"] == "openai"
assert decoded["response_id"] == "cntr_upstream_123"
@pytest.mark.asyncio
async def test_alist_containers_basic(self):
"""Test basic async container listing functionality."""