Merge f62059f720 into 80d3b69d9c

litellm/llms/hosted_vllm/chat/transformation.py

@@ -15,6 +15,9 @@ from typing import (
     cast,
     overload,
 )
+import hashlib
+import base64
+import secrets
 
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     _get_image_mime_type_from_url,
@@ -128,7 +131,32 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
         return super().map_openai_params(
             non_default_params, optional_params, model, drop_params
         )
+        
+    def transform_request(
+        self,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        request = super().transform_request(model, messages, optional_params, litellm_params, headers)
 
+        if request.get("cache_salt"):
+            return request
+
+        auth_header = headers.get("Authorization", "")
+        if auth_header.startswith("Bearer "):
+            api_key = auth_header[7:]
+        else:
+            api_key = ""
+        if api_key:
+            cache_salt = base64.b64encode(hashlib.sha256(api_key.encode()).digest()).decode()
+        else:
+            cache_salt = base64.b64encode(secrets.token_bytes(16)).decode()
+        request["cache_salt"] = cache_salt
+        return request
+        
     def _get_openai_compatible_provider_info(
         self, api_base: Optional[str], api_key: Optional[str]
     ) -> Tuple[Optional[str], Optional[str]]:

litellm/llms/hosted_vllm/responses/transformation.py

@@ -6,12 +6,17 @@ so this config enables direct routing instead of falling back to
 the chat completions → responses conversion pipeline.
 """
 
-from typing import Optional
+import base64
+import hashlib
+import secrets
+from typing import Dict, Optional, Union
 
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
+from litellm.types.llms.openai import ResponsesAPIRequestParams
+from litellm.types.llms.openai import ResponseInputParam
 
 
 class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig):
@@ -28,6 +33,35 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def custom_llm_provider(self) -> LlmProviders:
         return LlmProviders.HOSTED_VLLM
 
+    def transform_responses_api_request(
+        self,
+        model: str,
+        input: Union[str, ResponseInputParam],
+        response_api_optional_request_params: Dict,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Dict:
+        input = self._validate_input_param(input)
+        final_request_params = dict(
+            ResponsesAPIRequestParams(model=model, input=input, **response_api_optional_request_params)
+        )
+
+        if final_request_params.get("cache_salt"):
+            return final_request_params
+
+        metadata = getattr(litellm_params, "metadata", {}) or {}
+        caller_id = (
+            metadata.get("user_api_key_user_id")
+            or metadata.get("user_api_key_team_id")
+            or metadata.get("user_api_key_end_user_id")
+        )
+        if caller_id:
+            cache_salt = base64.b64encode(hashlib.sha256(caller_id.encode()).digest()).decode()
+        else:
+            cache_salt = base64.b64encode(secrets.token_bytes(16)).decode()
+        final_request_params["cache_salt"] = cache_salt
+        return final_request_params
+
     def validate_environment(
         self,
         headers: dict,