scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 124

fix(containers): module-level imports and managed cntr_ ID encoding

Browse source 
- Move ResponsesAPIRequestUtils imports to module scope (utils, main, handler_factory).
- Serialize absent model_id as empty segment instead of literal None; decode empty
  and legacy "None" segments as missing for router affinity.
- Add unit tests for build/decode round-trip and legacy IDs.

Made-with: Cursor
This commit is contained in:
Sameer Kankute 2026-04-07 19:28:46 +05:30
parent 91cb235401
commit 202ea11ad4
No known key found for this signature in database
5 changed files with 117 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
litellm/containers/main.py
View file

@ -7,6 +7,7 @@ from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, overloa
import litellm
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.containers.utils import ContainerRequestUtils
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.main import base_llm_http_handler
@ -50,8 +51,6 @@ def _decode_container_id_and_update_provider(
Returns:
tuple: (original_container_id, resolved_provider, updated_litellm_params)
"""
from litellm.responses.utils import ResponsesAPIRequestUtils
# Decode the container ID
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
@ -292,11 +291,11 @@ def create_container(
# Encode container_id with provider/model metadata for routing
if isinstance(container_obj, ContainerObject):
model_id = kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id")
container_obj = ContainerRequestUtils.encode_container_id_in_response(
response_obj=container_obj,
custom_llm_provider=custom_llm_provider,
model_id=model_id,
litellm_metadata=kwargs.get("litellm_metadata"),
extra_body=extra_body,
)
return container_obj
@ -660,6 +659,8 @@ def retrieve_container(
)
# Decode container ID and extract provider info
# Track if input was encoded so we can re-encode the output
was_encoded = container_id.startswith("cntr_") and len(container_id) > 100
original_container_id, custom_llm_provider, litellm_params = (
_decode_container_id_and_update_provider(
container_id=container_id,
@ -706,12 +707,23 @@ def retrieve_container(
)
# Encode container_id with provider/model metadata for routing
# If input was encoded, preserve encoding in output using the decoded model_id
if isinstance(container_obj, ContainerObject):
model_id = kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id")
# If input was encoded, use model_id from decoded params
litellm_metadata = kwargs.get("litellm_metadata", {})
if was_encoded and litellm_params.get("model_id"):
# Inject model_id from decoded container_id into litellm_metadata
if not litellm_metadata:
litellm_metadata = {}
if "model_info" not in litellm_metadata:
litellm_metadata["model_info"] = {}
litellm_metadata["model_info"]["id"] = litellm_params["model_id"]
container_obj = ContainerRequestUtils.encode_container_id_in_response(
response_obj=container_obj,
custom_llm_provider=custom_llm_provider,
model_id=model_id,
litellm_metadata=litellm_metadata,
extra_body=None,
)
return container_obj
@ -867,6 +879,8 @@ def delete_container(
)
# Decode container ID and extract provider info
# Track if input was encoded so we can re-encode the output
was_encoded = container_id.startswith("cntr_") and len(container_id) > 100
original_container_id, custom_llm_provider, litellm_params = (
_decode_container_id_and_update_provider(
container_id=container_id,
@ -913,12 +927,23 @@ def delete_container(
)
# Encode container_id in response with provider/model metadata for routing
# If input was encoded, preserve encoding in output using the decoded model_id
if isinstance(delete_result, DeleteContainerResult):
model_id = kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id")
# If input was encoded, use model_id from decoded params
litellm_metadata = kwargs.get("litellm_metadata", {})
if was_encoded and litellm_params.get("model_id"):
# Inject model_id from decoded container_id into litellm_metadata
if not litellm_metadata:
litellm_metadata = {}
if "model_info" not in litellm_metadata:
litellm_metadata["model_info"] = {}
litellm_metadata["model_info"]["id"] = litellm_params["model_id"]
delete_result = ContainerRequestUtils.encode_container_id_in_response(
response_obj=delete_result,
custom_llm_provider=custom_llm_provider,
model_id=model_id,
litellm_metadata=litellm_metadata,
extra_body=None,
)
return delete_result

52
litellm/containers/utils.py
View file

@ -1,6 +1,7 @@
from typing import Any, Dict, Optional, TypeVar
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
ContainerListOptionalRequestParams,
@ -75,30 +76,61 @@ class ContainerRequestUtils:
def encode_container_id_in_response(
response_obj: T,
custom_llm_provider: Optional[str],
model_id: Optional[str],
litellm_metadata: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
) -> T:
"""
Encode container_id in response object with provider/model metadata for routing.
This mirrors the responses API pattern where response IDs are encoded with
routing metadata so follow-up calls can route to the correct provider.
Encodes when:
1. litellm_metadata contains model_info.id (indicating router/proxy usage), OR
2. extra_body contains target_model_names (indicating model-specific routing)
Direct SDK calls with explicit custom_llm_provider and no routing hints return raw IDs.
Args:
response_obj: Response object with an `id` attribute (ContainerObject, DeleteContainerResult, etc.)
custom_llm_provider: Provider name (e.g., "azure", "openai")
model_id: Model ID from litellm_metadata
litellm_metadata: Optional litellm_metadata dict that may contain model_info.id
extra_body: Optional extra_body dict that may contain target_model_names
Returns:
The same response object with encoded container_id
The same response object with encoded container_id (if routing metadata present)
"""
if response_obj and hasattr(response_obj, "id"):
from litellm.responses.utils import ResponsesAPIRequestUtils
# Extract model_id from litellm_metadata
litellm_metadata = litellm_metadata or {}
model_info: Dict[str, Any] = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id")
# Check if we should encode based on routing metadata
should_encode = False
# Case 1: Router/proxy usage (model_id from router)
if model_id is not None:
should_encode = True
# Case 2: target_model_names in extra_body (model-specific routing)
if extra_body and "target_model_names" in extra_body:
should_encode = True
# Extract model_id from target_model_names if not already set
if model_id is None:
target_models = extra_body["target_model_names"]
# Use first model as model_id for encoding
if isinstance(target_models, str):
model_id = target_models.split(",")[0].strip()
elif isinstance(target_models, list) and len(target_models) > 0:
model_id = str(target_models[0]).strip()
# Only encode if we have routing metadata
if should_encode and response_obj and hasattr(response_obj, "id"):
encoded_id = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider=custom_llm_provider,
model_id=model_id,
container_id=response_obj.id,
)
response_obj.id = encoded_id
return response_obj

7
litellm/proxy/container_endpoints/handler_factory.py
View file

@ -19,6 +19,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
get_custom_llm_provider_from_request_headers,
get_custom_llm_provider_from_request_query,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
def _load_endpoints_config() -> Dict:
@ -178,8 +179,6 @@ async def _process_binary_request(
litellm_params = GenericLiteLLMParams()
# Decode container ID and extract provider info
from litellm.responses.utils import ResponsesAPIRequestUtils
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
original_container_id = decoded.get("response_id", container_id)
@ -288,8 +287,6 @@ async def _process_multipart_upload_request(
)
# Decode container ID and extract provider info
from litellm.responses.utils import ResponsesAPIRequestUtils
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
original_container_id = decoded.get("response_id", container_id)
@ -366,8 +363,6 @@ async def _process_request(
# Decode container_id if present in path_params
if "container_id" in path_params:
from litellm.responses.utils import ResponsesAPIRequestUtils
decoded = ResponsesAPIRequestUtils._decode_container_id(
path_params["container_id"]
)

16
litellm/responses/utils.py
View file

@ -542,7 +542,10 @@ class ResponsesAPIRequestUtils:
Format: cntr_{base64("litellm:custom_llm_provider:{provider};model_id:{model};container_id:{original}")}
"""
assembled_id = f"litellm:custom_llm_provider:{custom_llm_provider};model_id:{model_id};container_id:{container_id}"
# Avoid serializing Python None as the literal string "None" (breaks router affinity).
provider_part = "" if custom_llm_provider is None else custom_llm_provider
model_part = "" if model_id is None else model_id
assembled_id = f"litellm:custom_llm_provider:{provider_part};model_id:{model_part};container_id:{container_id}"
base64_encoded_id = base64.b64encode(assembled_id.encode("utf-8")).decode("utf-8")
return f"cntr_{base64_encoded_id}"
@ -576,7 +579,8 @@ class ResponsesAPIRequestUtils:
# Use regex to extract the three parts, allowing semicolons in container_id
# Format: litellm:custom_llm_provider:{provider};model_id:{model};container_id:{container}
pattern = r"^litellm:custom_llm_provider:([^;]+);model_id:([^;]+);container_id:(.+)$"
# * for provider/model allows empty segments (missing router model_id).
pattern = r"^litellm:custom_llm_provider:([^;]*);model_id:([^;]*);container_id:(.+)$"
match = re.match(pattern, decoded_id)
if not match:
@ -586,8 +590,12 @@ class ResponsesAPIRequestUtils:
response_id=container_id,
)
custom_llm_provider = match.group(1)
model_id = match.group(2)
raw_provider = match.group(1)
raw_model_id = match.group(2)
custom_llm_provider = (
None if raw_provider in ("", "None") else raw_provider
)
model_id = None if raw_model_id in ("", "None") else raw_model_id
original_container_id = match.group(3)
return DecodedResponseId(

29
tests/test_litellm/responses/test_responses_utils.py
View file

@ -138,6 +138,35 @@ class TestResponsesAPIRequestUtils:
assert decoded.get("model_id") == "gpt-4o"
assert decoded.get("custom_llm_provider") == "openai"
def test_build_decode_container_id_omits_none_model_id(self):
"""model_id=None must not round-trip as the truthy string 'None'."""
encoded = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider="azure",
model_id=None,
container_id="cntr_upstream_abc",
)
assert "None" not in base64.b64decode(
encoded.replace("cntr_", "").encode("utf-8")
).decode("utf-8")
decoded = ResponsesAPIRequestUtils._decode_container_id(encoded)
assert decoded.get("custom_llm_provider") == "azure"
assert decoded.get("model_id") is None
assert decoded.get("response_id") == "cntr_upstream_abc"
def test_decode_container_id_legacy_literal_none_model_id(self):
"""IDs encoded before the None fix should decode without a bogus model_id."""
legacy_inner = (
"litellm:custom_llm_provider:azure;model_id:None;container_id:cntr_x"
)
legacy_id = (
"cntr_"
+ base64.b64encode(legacy_inner.encode("utf-8")).decode("utf-8")
)
decoded = ResponsesAPIRequestUtils._decode_container_id(legacy_id)
assert decoded.get("model_id") is None
assert decoded.get("custom_llm_provider") == "azure"
assert decoded.get("response_id") == "cntr_x"
class TestResponseAPILoggingUtils:
def test_is_response_api_usage_true(self):