diff --git a/litellm/containers/main.py b/litellm/containers/main.py index ccdc0087fae..5b72dfb9d41 100644 --- a/litellm/containers/main.py +++ b/litellm/containers/main.py @@ -7,6 +7,7 @@ from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, overloa import litellm from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT from litellm.containers.utils import ContainerRequestUtils +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.containers.transformation import BaseContainerConfig from litellm.main import base_llm_http_handler @@ -50,8 +51,6 @@ def _decode_container_id_and_update_provider( Returns: tuple: (original_container_id, resolved_provider, updated_litellm_params) """ - from litellm.responses.utils import ResponsesAPIRequestUtils - # Decode the container ID decoded = ResponsesAPIRequestUtils._decode_container_id(container_id) @@ -292,11 +291,11 @@ def create_container( # Encode container_id with provider/model metadata for routing if isinstance(container_obj, ContainerObject): - model_id = kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id") container_obj = ContainerRequestUtils.encode_container_id_in_response( response_obj=container_obj, custom_llm_provider=custom_llm_provider, - model_id=model_id, + litellm_metadata=kwargs.get("litellm_metadata"), + extra_body=extra_body, ) return container_obj @@ -660,6 +659,8 @@ def retrieve_container( ) # Decode container ID and extract provider info + # Track if input was encoded so we can re-encode the output + was_encoded = container_id.startswith("cntr_") and len(container_id) > 100 original_container_id, custom_llm_provider, litellm_params = ( _decode_container_id_and_update_provider( container_id=container_id, @@ -706,12 +707,23 @@ def retrieve_container( ) # Encode container_id with provider/model metadata for routing + # If input was encoded, preserve encoding in output using the decoded model_id if isinstance(container_obj, ContainerObject): - model_id = kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id") + # If input was encoded, use model_id from decoded params + litellm_metadata = kwargs.get("litellm_metadata", {}) + if was_encoded and litellm_params.get("model_id"): + # Inject model_id from decoded container_id into litellm_metadata + if not litellm_metadata: + litellm_metadata = {} + if "model_info" not in litellm_metadata: + litellm_metadata["model_info"] = {} + litellm_metadata["model_info"]["id"] = litellm_params["model_id"] + container_obj = ContainerRequestUtils.encode_container_id_in_response( response_obj=container_obj, custom_llm_provider=custom_llm_provider, - model_id=model_id, + litellm_metadata=litellm_metadata, + extra_body=None, ) return container_obj @@ -867,6 +879,8 @@ def delete_container( ) # Decode container ID and extract provider info + # Track if input was encoded so we can re-encode the output + was_encoded = container_id.startswith("cntr_") and len(container_id) > 100 original_container_id, custom_llm_provider, litellm_params = ( _decode_container_id_and_update_provider( container_id=container_id, @@ -913,12 +927,23 @@ def delete_container( ) # Encode container_id in response with provider/model metadata for routing + # If input was encoded, preserve encoding in output using the decoded model_id if isinstance(delete_result, DeleteContainerResult): - model_id = kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id") + # If input was encoded, use model_id from decoded params + litellm_metadata = kwargs.get("litellm_metadata", {}) + if was_encoded and litellm_params.get("model_id"): + # Inject model_id from decoded container_id into litellm_metadata + if not litellm_metadata: + litellm_metadata = {} + if "model_info" not in litellm_metadata: + litellm_metadata["model_info"] = {} + litellm_metadata["model_info"]["id"] = litellm_params["model_id"] + delete_result = ContainerRequestUtils.encode_container_id_in_response( response_obj=delete_result, custom_llm_provider=custom_llm_provider, - model_id=model_id, + litellm_metadata=litellm_metadata, + extra_body=None, ) return delete_result diff --git a/litellm/containers/utils.py b/litellm/containers/utils.py index 43efb452f3f..e707109d50b 100644 --- a/litellm/containers/utils.py +++ b/litellm/containers/utils.py @@ -1,6 +1,7 @@ from typing import Any, Dict, Optional, TypeVar from litellm.llms.base_llm.containers.transformation import BaseContainerConfig +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.containers.main import ( ContainerCreateOptionalRequestParams, ContainerListOptionalRequestParams, @@ -75,30 +76,61 @@ class ContainerRequestUtils: def encode_container_id_in_response( response_obj: T, custom_llm_provider: Optional[str], - model_id: Optional[str], + litellm_metadata: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, ) -> T: """ Encode container_id in response object with provider/model metadata for routing. - + This mirrors the responses API pattern where response IDs are encoded with routing metadata so follow-up calls can route to the correct provider. - + + Encodes when: + 1. litellm_metadata contains model_info.id (indicating router/proxy usage), OR + 2. extra_body contains target_model_names (indicating model-specific routing) + + Direct SDK calls with explicit custom_llm_provider and no routing hints return raw IDs. + Args: response_obj: Response object with an `id` attribute (ContainerObject, DeleteContainerResult, etc.) custom_llm_provider: Provider name (e.g., "azure", "openai") - model_id: Model ID from litellm_metadata - + litellm_metadata: Optional litellm_metadata dict that may contain model_info.id + extra_body: Optional extra_body dict that may contain target_model_names + Returns: - The same response object with encoded container_id + The same response object with encoded container_id (if routing metadata present) """ - if response_obj and hasattr(response_obj, "id"): - from litellm.responses.utils import ResponsesAPIRequestUtils - + # Extract model_id from litellm_metadata + litellm_metadata = litellm_metadata or {} + model_info: Dict[str, Any] = litellm_metadata.get("model_info", {}) or {} + model_id = model_info.get("id") + + # Check if we should encode based on routing metadata + should_encode = False + + # Case 1: Router/proxy usage (model_id from router) + if model_id is not None: + should_encode = True + + # Case 2: target_model_names in extra_body (model-specific routing) + if extra_body and "target_model_names" in extra_body: + should_encode = True + # Extract model_id from target_model_names if not already set + if model_id is None: + target_models = extra_body["target_model_names"] + # Use first model as model_id for encoding + if isinstance(target_models, str): + model_id = target_models.split(",")[0].strip() + elif isinstance(target_models, list) and len(target_models) > 0: + model_id = str(target_models[0]).strip() + + # Only encode if we have routing metadata + if should_encode and response_obj and hasattr(response_obj, "id"): encoded_id = ResponsesAPIRequestUtils._build_container_id( custom_llm_provider=custom_llm_provider, model_id=model_id, container_id=response_obj.id, ) response_obj.id = encoded_id - + return response_obj diff --git a/litellm/proxy/container_endpoints/handler_factory.py b/litellm/proxy/container_endpoints/handler_factory.py index badada9439e..f2b23ff95b0 100644 --- a/litellm/proxy/container_endpoints/handler_factory.py +++ b/litellm/proxy/container_endpoints/handler_factory.py @@ -19,6 +19,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( get_custom_llm_provider_from_request_headers, get_custom_llm_provider_from_request_query, ) +from litellm.responses.utils import ResponsesAPIRequestUtils def _load_endpoints_config() -> Dict: @@ -178,8 +179,6 @@ async def _process_binary_request( litellm_params = GenericLiteLLMParams() # Decode container ID and extract provider info - from litellm.responses.utils import ResponsesAPIRequestUtils - decoded = ResponsesAPIRequestUtils._decode_container_id(container_id) original_container_id = decoded.get("response_id", container_id) @@ -288,8 +287,6 @@ async def _process_multipart_upload_request( ) # Decode container ID and extract provider info - from litellm.responses.utils import ResponsesAPIRequestUtils - decoded = ResponsesAPIRequestUtils._decode_container_id(container_id) original_container_id = decoded.get("response_id", container_id) @@ -366,8 +363,6 @@ async def _process_request( # Decode container_id if present in path_params if "container_id" in path_params: - from litellm.responses.utils import ResponsesAPIRequestUtils - decoded = ResponsesAPIRequestUtils._decode_container_id( path_params["container_id"] ) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index a617b7e2e07..bc9fe3897a3 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -542,7 +542,10 @@ class ResponsesAPIRequestUtils: Format: cntr_{base64("litellm:custom_llm_provider:{provider};model_id:{model};container_id:{original}")} """ - assembled_id = f"litellm:custom_llm_provider:{custom_llm_provider};model_id:{model_id};container_id:{container_id}" + # Avoid serializing Python None as the literal string "None" (breaks router affinity). + provider_part = "" if custom_llm_provider is None else custom_llm_provider + model_part = "" if model_id is None else model_id + assembled_id = f"litellm:custom_llm_provider:{provider_part};model_id:{model_part};container_id:{container_id}" base64_encoded_id = base64.b64encode(assembled_id.encode("utf-8")).decode("utf-8") return f"cntr_{base64_encoded_id}" @@ -576,7 +579,8 @@ class ResponsesAPIRequestUtils: # Use regex to extract the three parts, allowing semicolons in container_id # Format: litellm:custom_llm_provider:{provider};model_id:{model};container_id:{container} - pattern = r"^litellm:custom_llm_provider:([^;]+);model_id:([^;]+);container_id:(.+)$" + # * for provider/model allows empty segments (missing router model_id). + pattern = r"^litellm:custom_llm_provider:([^;]*);model_id:([^;]*);container_id:(.+)$" match = re.match(pattern, decoded_id) if not match: @@ -586,8 +590,12 @@ class ResponsesAPIRequestUtils: response_id=container_id, ) - custom_llm_provider = match.group(1) - model_id = match.group(2) + raw_provider = match.group(1) + raw_model_id = match.group(2) + custom_llm_provider = ( + None if raw_provider in ("", "None") else raw_provider + ) + model_id = None if raw_model_id in ("", "None") else raw_model_id original_container_id = match.group(3) return DecodedResponseId( diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index c6f32b6d758..33f354f444f 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -138,6 +138,35 @@ class TestResponsesAPIRequestUtils: assert decoded.get("model_id") == "gpt-4o" assert decoded.get("custom_llm_provider") == "openai" + def test_build_decode_container_id_omits_none_model_id(self): + """model_id=None must not round-trip as the truthy string 'None'.""" + encoded = ResponsesAPIRequestUtils._build_container_id( + custom_llm_provider="azure", + model_id=None, + container_id="cntr_upstream_abc", + ) + assert "None" not in base64.b64decode( + encoded.replace("cntr_", "").encode("utf-8") + ).decode("utf-8") + decoded = ResponsesAPIRequestUtils._decode_container_id(encoded) + assert decoded.get("custom_llm_provider") == "azure" + assert decoded.get("model_id") is None + assert decoded.get("response_id") == "cntr_upstream_abc" + + def test_decode_container_id_legacy_literal_none_model_id(self): + """IDs encoded before the None fix should decode without a bogus model_id.""" + legacy_inner = ( + "litellm:custom_llm_provider:azure;model_id:None;container_id:cntr_x" + ) + legacy_id = ( + "cntr_" + + base64.b64encode(legacy_inner.encode("utf-8")).decode("utf-8") + ) + decoded = ResponsesAPIRequestUtils._decode_container_id(legacy_id) + assert decoded.get("model_id") is None + assert decoded.get("custom_llm_provider") == "azure" + assert decoded.get("response_id") == "cntr_x" + class TestResponseAPILoggingUtils: def test_is_response_api_usage_true(self):