diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 7c42e6a9a00..13e0fb6261e 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -15,6 +15,9 @@ from typing import ( cast, overload, ) +import hashlib +import base64 +import secrets from litellm.litellm_core_utils.prompt_templates.common_utils import ( _get_image_mime_type_from_url, @@ -128,7 +131,32 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): return super().map_openai_params( non_default_params, optional_params, model, drop_params ) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + request = super().transform_request(model, messages, optional_params, litellm_params, headers) + if request.get("cache_salt"): + return request + + auth_header = headers.get("Authorization", "") + if auth_header.startswith("Bearer "): + api_key = auth_header[7:] + else: + api_key = "" + if api_key: + cache_salt = base64.b64encode(hashlib.sha256(api_key.encode()).digest()).decode() + else: + cache_salt = base64.b64encode(secrets.token_bytes(16)).decode() + request["cache_salt"] = cache_salt + return request + def _get_openai_compatible_provider_info( self, api_base: Optional[str], api_key: Optional[str] ) -> Tuple[Optional[str], Optional[str]]: diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 4d44eeda9f9..f8a02fad076 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -6,12 +6,17 @@ so this config enables direct routing instead of falling back to the chat completions → responses conversion pipeline. """ -from typing import Optional +import base64 +import hashlib +import secrets +from typing import Dict, Optional, Union from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.types.llms.openai import ResponsesAPIRequestParams +from litellm.types.llms.openai import ResponseInputParam class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -28,6 +33,35 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.HOSTED_VLLM + def transform_responses_api_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + input = self._validate_input_param(input) + final_request_params = dict( + ResponsesAPIRequestParams(model=model, input=input, **response_api_optional_request_params) + ) + + if final_request_params.get("cache_salt"): + return final_request_params + + metadata = getattr(litellm_params, "metadata", {}) or {} + caller_id = ( + metadata.get("user_api_key_user_id") + or metadata.get("user_api_key_team_id") + or metadata.get("user_api_key_end_user_id") + ) + if caller_id: + cache_salt = base64.b64encode(hashlib.sha256(caller_id.encode()).digest()).decode() + else: + cache_salt = base64.b64encode(secrets.token_bytes(16)).decode() + final_request_params["cache_salt"] = cache_salt + return final_request_params + def validate_environment( self, headers: dict,