diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9b8515a9ecd..5099352beff 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -15,7 +15,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( openai_prompt_cache_breakpoint_support, targets_openai_api, ) -from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers +from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, process_response_headers from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( safe_convert_created_field, @@ -44,6 +44,7 @@ from litellm.responses.litellm_completion_transformation.tool_search import ( tools_with_anthropic_tool_search_mapped, tools_without_anthropic_tool_search, ) +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import * @@ -623,11 +624,45 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): "model": model, "input": replay_safe_input, **response_api_optional_request_params, + **self._prompt_cache_options_with_original_comparison_id(request_params), }, ) return final_request_params + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: GenericLiteLLMParams, + ) -> Mapping[str, object]: + return {**extra_body, **self._prompt_cache_options_with_original_comparison_id(extra_body)} + + def _prompt_cache_options_with_original_comparison_id(self, params: Mapping[str, object]) -> Mapping[str, object]: + options: Final = as_str_mapping(params.get("prompt_cache_options")) + comparison_response_id: Final = None if options is None else options.get("comparison_response_id") + if ( + self.custom_llm_provider != LlmProviders.OPENAI + or options is None + or not isinstance(comparison_response_id, str) + or not comparison_response_id.startswith("resp_") + ): + return {} + decoded_id: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(comparison_response_id) + decoded_provider: Final = decoded_id.get("custom_llm_provider") + original_response_id: Final = decoded_id.get("response_id", comparison_response_id) + rebuilt_envelope: Final = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( + {"id": original_response_id}, decoded_provider, {"model_info": {"id": decoded_id.get("model_id")}} + ) + if ( + decoded_provider != LlmProviders.OPENAI + or original_response_id == comparison_response_id + or rebuilt_envelope.get("id") != comparison_response_id + ): + return {} + return {"prompt_cache_options": {**options, "comparison_response_id": original_response_id}} + def _prepared_input_and_tools( self, model: str, diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 771d68a0ea0..ef03385e065 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -14,6 +14,7 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import LitellmUserRoles from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, @@ -23,7 +24,13 @@ from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, ResponsesAPIResponse, ) -from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypesLiteral, LLMResponseTypes, SpecialEnums +from litellm.types.utils import ( + ADDRESSED_COMPARISON_RESPONSE_ID_FIELD, + ADDRESSED_RESPONSE_ID_FIELD, + CallTypesLiteral, + LLMResponseTypes, + SpecialEnums, +) if TYPE_CHECKING: from litellm.caching.caching import DualCache @@ -135,6 +142,9 @@ class ResponsesIDSecurity(CustomLogger): addressed_id_field: Final = _ADDRESSED_RESPONSE_ID_FIELD_BY_CALL_TYPE.get(call_type) if addressed_id_field is None: return None + if call_type == "aresponses": + request_data: Final = cast("dict[str, object]", data) # cast-ok: the hook signature leaves the body untyped + request_data.update(self._authorized_comparison_response_id_fields(request_data, user_api_key_dict)) retained_id: Final = data.get(ADDRESSED_RESPONSE_ID_FIELD) addressed_id: Final = ( retained_id if isinstance(retained_id, str) and retained_id else data.get(addressed_id_field) @@ -146,6 +156,39 @@ class ResponsesIDSecurity(CustomLogger): data[ADDRESSED_RESPONSE_ID_FIELD] = addressed_id return data + def _authorized_comparison_response_id_fields( + self, + data: Mapping[str, object], + user_api_key_dict: "UserAPIKeyAuth", + ) -> Mapping[str, object]: + extra_body: Final = as_str_mapping(data.get("extra_body")) + overriding_extra_body: Final = ( + extra_body if extra_body is not None and "prompt_cache_options" in extra_body else None + ) + options: Final = as_str_mapping( + data.get("prompt_cache_options") + if overriding_extra_body is None + else overriding_extra_body.get("prompt_cache_options") + ) + retained_id: Final = data.get(ADDRESSED_COMPARISON_RESPONSE_ID_FIELD) + requested_id: Final = None if options is None else options.get("comparison_response_id") + if options is None or not isinstance(requested_id, str): + return {} + addressed_id: Final = requested_id if self._is_encrypted_response_id(requested_id) else retained_id + if not isinstance(addressed_id, str) or not self._is_encrypted_response_id(addressed_id): + return {} + original_response_id, user_id, team_id = self._decrypt_response_id(addressed_id) + if requested_id not in (addressed_id, original_response_id): + return {} + self.check_user_access_to_response_id(user_id, team_id, user_api_key_dict) + authorized_options: Final = {**options, "comparison_response_id": original_response_id} + authorized_field: Final[Mapping[str, object]] = ( + {"prompt_cache_options": authorized_options} + if overriding_extra_body is None + else {"extra_body": {**overriding_extra_body, "prompt_cache_options": authorized_options}} + ) + return {**authorized_field, ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: addressed_id} + def authorize_response_id( self, response_id: str, diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 9f1b35fdce3..b6b17a6b02e 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -55,6 +55,7 @@ RoutingStrategyName: TypeAlias = Literal[ TRUSTED_CALLBACK_VARS_FIELD: Final = "litellm_trusted_callback_vars" ADDRESSED_RESPONSE_ID_FIELD: Final = "_litellm_addressed_response_id" +ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: Final = "_litellm_addressed_comparison_response_id" STREAM_USAGE_INJECTED_FIELD: Final = "_litellm_stream_usage_injected" WIRE_NAME: Final = "wire_name" @@ -406,6 +407,9 @@ class ProxyRequestState: secret_fields: "SecretFields | None" = None trusted_callback_vars: Mapping[str, str] | None = field(default=None, metadata=wire(TRUSTED_CALLBACK_VARS_FIELD)) addressed_response_id: str | None = field(default=None, metadata=wire(ADDRESSED_RESPONSE_ID_FIELD)) + addressed_comparison_response_id: str | None = field( + default=None, metadata=wire(ADDRESSED_COMPARISON_RESPONSE_ID_FIELD) + ) strip_stream_usage: bool | None = field(default=None, metadata=wire("_litellm_strip_stream_usage")) stream_usage_injected: bool | None = field(default=None, metadata=wire(STREAM_USAGE_INJECTED_FIELD)) client_side_timeout: bool | None = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a54d66c4beb..cde0db6fac8 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -689,6 +689,7 @@ class PromptCacheBreakpoint(TypedDict): class PromptCacheOptions(TypedDict, total=False): mode: ReadOnly[Literal["implicit", "explicit"]] ttl: ReadOnly[Literal["30m", "12h"]] + comparison_response_id: ReadOnly[str] class ChatCompletionThinkingBlock(TypedDict, total=False): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0c3440e88f7..a301a5fb2af 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4334,6 +4334,7 @@ bedrock_batch_litellm_params: Final = BEDROCK_BATCH_KWARG_NAMES TRUSTED_CALLBACK_VARS_FIELD: Final = _litellm_params.TRUSTED_CALLBACK_VARS_FIELD ADDRESSED_RESPONSE_ID_FIELD: Final = _litellm_params.ADDRESSED_RESPONSE_ID_FIELD +ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: Final = _litellm_params.ADDRESSED_COMPARISON_RESPONSE_ID_FIELD anthropic_wif_litellm_params: Final = tuple(sorted(ANTHROPIC_WIF_KWARGS_KEYS)) openai_wif_litellm_params: Final = tuple(sorted(OPENAI_WIF_KWARGS_KEYS)) diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index a9098ac3d59..798f6ab2ea6 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -9269,6 +9269,7 @@ PROXY_STAMPED_NAMES: Final = frozenset( "secret_fields", "litellm_trusted_callback_vars", "_litellm_addressed_response_id", + "_litellm_addressed_comparison_response_id", "_litellm_strip_stream_usage", "_litellm_stream_usage_injected", "client_side_timeout", diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 22eb9929e7f..0cccca4ba57 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -24,6 +24,7 @@ from litellm.types import litellm_params from litellm.types import utils as types_utils from litellm.types.caching import DynamicCacheControl from litellm.types.litellm_params import ( + ADDRESSED_COMPARISON_RESPONSE_ID_FIELD, ADDRESSED_RESPONSE_ID_FIELD, LITELLM_OWNED_ROOTS, TRUSTED_CALLBACK_VARS_FIELD, @@ -269,6 +270,7 @@ INTERNAL_STATE_NAMES: Final = ( "github_copilot_user_session", "litellm_trusted_callback_vars", "_litellm_addressed_response_id", + "_litellm_addressed_comparison_response_id", "_litellm_strip_stream_usage", "_litellm_stream_usage_injected", "client_side_timeout", @@ -458,9 +460,10 @@ def test_bedrock_batch_names_concatenate_as_a_tuple() -> None: def test_proxy_stamped_fields_keep_their_wire_names() -> None: - assert (TRUSTED_CALLBACK_VARS_FIELD, ADDRESSED_RESPONSE_ID_FIELD) == ( + assert (TRUSTED_CALLBACK_VARS_FIELD, ADDRESSED_RESPONSE_ID_FIELD, ADDRESSED_COMPARISON_RESPONSE_ID_FIELD) == ( "litellm_trusted_callback_vars", "_litellm_addressed_response_id", + "_litellm_addressed_comparison_response_id", ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 30d61f181cb..1336949c2fb 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -41886,6 +41886,8 @@ export interface components { }; /** PromptCacheOptions */ PromptCacheOptions: { + /** Comparison Response Id */ + comparison_response_id?: string; /** * Mode * @enum {string}