From a844dadcd27ececec63eca6c8bf3a5b819353f55 Mon Sep 17 00:00:00 2001 From: Aryan Pardeshi Date: Fri, 16 Oct 2026 14:31:33 +0530 Subject: [PATCH] feat(gemini): allow disabling thought summaries while keeping thinking on (#36460) * feat(gemini): allow disabling thought summaries while keeping thinking on Both reasoning-effort mappers hardcoded includeThoughts to True whenever thinking was on, so thinking ON with summaries OFF was unreachable even though that is the google SDK's own default. Summaries are the narration rather than the reasoning, and for callers that discard them they are pure added latency. Adds litellm.gemini_include_thoughts_default, defaulting to True so nothing changes for anyone who does not set it. It gates the includeThoughts=True branches in the thinkingBudget mapper, the thinkingLevel mapper, and the anthropic-style thinking param mapper. The disable and none paths still force includeThoughts=False: with thinking off there is nothing to summarise. * test(gemini): assert thought-summary flag through the outbound Gemini request * feat(gemini): let a request's thought summary ask override the gateway default thinking.display (summarized, updates, omitted) and a reasoning_effort object carrying summary now decide includeThoughts for that request; gemini_include_thoughts_default applies only when the request says nothing * feat(gemini): let a thought summary ask on /v1/messages and /v1/responses override the gateway default * fix(gemini): read a text gateway default as a boolean and leave display omitted to the flag Normalize gemini_include_thoughts_default through normalize_drop_params so a YAML or env value written as "false" or "0" turns thought text off and a null or unknown value keeps the default on. thinking.display "omitted" no longer counts as a per-request ask, so the gateway flag decides, matching /v1/messages. Adds proxy integration cells for chat, /v1/messages and /v1/responses that prove the flag, every per-request ask, streaming, and a burst across workers. * test(gemini): match the streamed Responses usage, which omits null token details * feat(gemini): read the AI SDK reasoning summary alias and thinking.summary as chat asks * fix(gemini): keep the logged reasoning_effort unchanged and key the response cache on the summary alias A dict reasoning_effort next to the AI SDK reasoning_summary alias now counts as a summary ask, and the alias is part of the cache key so a plain request and an alias ask never share a reply * fix(caching): key the response cache on the summary alias wherever the request carries it The cache key now reads reasoning_summary and reasoningSummary through the same reader as the summary fold, so an alias inside extra_body gets its own key too * test: type the new summary-ask unit tests and mark their locals Final * test: annotate the remaining summary-ask unit tests the PR touches --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 1 + litellm/caching/caching.py | 13 +- .../litellm_core_utils/model_param_helper.py | 19 + .../pass_through/adapters/handler.py | 52 +- litellm/llms/base_llm/chat/transformation.py | 4 + .../vertex_and_google_ai_studio_gemini.py | 70 +- litellm/main.py | 11 +- litellm/responses/bridge_check.py | 16 +- .../transformation.py | 10 +- litellm/utils.py | 32 +- .../chat_completions/reasoning/__init__.py | 0 .../test_chat_completions_reasoning_gemini.py | 1041 +++++++++++++++ .../translation/gemini_thoughts.py | 63 + .../test_messages_reasoning_gemini.py | 373 ++++++ .../responses/reasoning/__init__.py | 0 .../test_responses_reasoning_gemini.py | 400 ++++++ tests/unit/caching/test_caching.py | 152 +-- ..._handler_reasoning_effort_normalization.py | 103 +- ...test_vertex_and_google_ai_studio_gemini.py | 1114 +++++++++-------- .../test_litellm_completion_responses.py | 41 +- 20 files changed, 2841 insertions(+), 674 deletions(-) create mode 100644 tests/integration/translation/chat_completions/reasoning/__init__.py create mode 100644 tests/integration/translation/chat_completions/reasoning/test_chat_completions_reasoning_gemini.py create mode 100644 tests/integration/translation/gemini_thoughts.py create mode 100644 tests/integration/translation/messages/reasoning/test_messages_reasoning_gemini.py create mode 100644 tests/integration/translation/responses/reasoning/__init__.py create mode 100644 tests/integration/translation/responses/reasoning/test_responses_reasoning_gemini.py diff --git a/litellm/__init__.py b/litellm/__init__.py index ff901ede512..af812dade3c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -393,6 +393,7 @@ key_alias_pattern: str | None = None enable_gemini_default_thinking_level_low: bool = ( False # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping ) +gemini_include_thoughts_default: bool = True #################### logging: bool = True enable_loadbalancing_on_batch_endpoints: Optional[bool] = None diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 655131e3c69..5f5b8210096 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -25,7 +25,7 @@ from litellm._internal_context import current_service_target, service_target from litellm._logging import verbose_logger from litellm.constants import CACHED_STREAMING_CHUNK_DELAY from litellm.integrations.otel.runtime import phase_span -from litellm.litellm_core_utils.model_param_helper import ModelParamHelper +from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases from litellm.types.caching import * from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg @@ -376,7 +376,7 @@ class Cache: ) return "".join(f"{field}: {value}" for field, value in scope_values if value is not None) - def get_cache_key(self, **kwargs) -> str: + def get_cache_key(self, **kwargs: object) -> str: """ Get the cache key for the given arguments. @@ -410,13 +410,16 @@ class Cache: if litellm.enable_caching_on_provider_specific_optional_params is True: # feature flagged for now if kwargs[param] is None: continue # ignore None params - param_value = kwargs[param] - cache_key += f"{param}: {param_value}" + cache_key += f"{param}: {kwargs[param]}" if is_semantic_cache: cache_key += self._get_semantic_cache_tenant_scope(kwargs) - hashed_cache_key = Cache._get_hashed_cache_key(cache_key) + reasoning_summary: Final = peek_reasoning_summary_aliases(kwargs) + reasoning_summary_material: Final = ( + "" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}" + ) + hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material) hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs) verbose_logger.debug( "\nCreated cache key: %s (source material length: %d)", diff --git a/litellm/litellm_core_utils/model_param_helper.py b/litellm/litellm_core_utils/model_param_helper.py index 2dc52980875..c9c7a0e89ac 100644 --- a/litellm/litellm_core_utils/model_param_helper.py +++ b/litellm/litellm_core_utils/model_param_helper.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from functools import lru_cache from typing import ClassVar, Final @@ -22,6 +23,24 @@ from litellm.types.llms.anthropic import AnthropicMessagesRequest from litellm.types.rerank import RerankRequest +def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> object | None: + """Read AI-SDK-style reasoning summary from optional_params or nested extra_body. + + Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped. + """ + if "reasoningSummary" in optional_params: + return optional_params["reasoningSummary"] + if "reasoning_summary" in optional_params: + return optional_params["reasoning_summary"] + extra_body: Final = optional_params.get("extra_body") + if isinstance(extra_body, dict): + if "reasoningSummary" in extra_body: + return extra_body["reasoningSummary"] + if "reasoning_summary" in extra_body: + return extra_body["reasoning_summary"] + return None + + class ModelParamHelper: # Cached at class level — deterministic set built from static OpenAI type annotations relevant_logging_args: ClassVar[frozenset[str]] = frozenset() diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 4c0613777c6..60c63964492 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -1,4 +1,5 @@ from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from dataclasses import dataclass from typing import ( TYPE_CHECKING, Final, @@ -62,6 +63,12 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object): custom_llm_provider: str +@dataclass(frozen=True, slots=True) +class _ChatTargetReasoning: + effort: str + summary: str | None = None + + def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool: """Return True when any message carries a ``compaction`` content block.""" for msg in messages: @@ -441,15 +448,33 @@ class LiteLLMMessagesToCompletionTransformationHandler: completion_kwargs["reasoning_effort"] = updated_reasoning_effort @staticmethod - def _plain_effort_for_chat_target( + def _requested_reasoning_summary(thinking: Mapping[str, object]) -> str | None: + if thinking.get("type") == "disabled": + return None + summary: Final = thinking.get("summary") + if isinstance(summary, str): + return summary + display: Final = thinking.get("display") + return "auto" if display is not None and display != "omitted" else None + + @staticmethod + def _chat_target_reasoning( completion_kwargs: _CompletionKwargs, *, thinking: Mapping[str, object] | None, - ) -> str | None: - reasoning_effort: Final = completion_kwargs.get("reasoning_effort") - if not thinking or not isinstance(reasoning_effort, dict) or "summary" not in reasoning_effort: + ) -> _ChatTargetReasoning | None: + if not thinking: return None - effort: Final = reasoning_effort.get("effort") + reasoning_effort: Final = completion_kwargs.get("reasoning_effort") + wraps_summary: Final = isinstance(reasoning_effort, dict) and "summary" in reasoning_effort + requested_summary: Final = LiteLLMMessagesToCompletionTransformationHandler._requested_reasoning_summary( + thinking + ) + if not wraps_summary and requested_summary is None: + return None + effort: Final[object] = ( + reasoning_effort.get("effort") if isinstance(reasoning_effort, dict) else reasoning_effort + ) model: Final = completion_kwargs.get("model") if not isinstance(effort, str) or not isinstance(model, str) or not model: return None @@ -467,7 +492,12 @@ class LiteLLMMessagesToCompletionTransformationHandler: return None if resolved_provider == "litellm_proxy": return None - from litellm.responses.bridge_check import responses_api_bridge_check + from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary, responses_api_bridge_check + + if requested_summary is not None and chat_target_accepts_reasoning_summary(local_model, resolved_provider): + return _ChatTargetReasoning(effort=effort, summary=requested_summary) + if not wraps_summary or not isinstance(reasoning_effort, dict): + return None web_search_options: Final = completion_kwargs.get("web_search_options") tools: Final = completion_kwargs.get("tools") @@ -481,7 +511,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: reasoning_effort=reasoning_effort, api_base=resolved_api_base, ) - return None if model_info.get("mode") == "responses" else effort + return None if model_info.get("mode") == "responses" else _ChatTargetReasoning(effort=effort) @staticmethod def _normalize_reasoning_effort( @@ -647,12 +677,14 @@ class LiteLLMMessagesToCompletionTransformationHandler: thinking=thinking, ) - plain_effort: Final = LiteLLMMessagesToCompletionTransformationHandler._plain_effort_for_chat_target( + chat_reasoning: Final = LiteLLMMessagesToCompletionTransformationHandler._chat_target_reasoning( completion_kwargs, thinking=thinking, ) - if plain_effort is not None: - completion_kwargs["reasoning_effort"] = plain_effort + if chat_reasoning is not None: + completion_kwargs["reasoning_effort"] = chat_reasoning.effort + if chat_reasoning is not None and chat_reasoning.summary is not None: + completion_kwargs["reasoning_summary"] = chat_reasoning.summary return completion_kwargs, tool_name_mapping diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 2b3c644f266..db4d5e0f820 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -488,6 +488,10 @@ class BaseConfig(ABC): """ return True + @property + def accepts_reasoning_summary_in_reasoning_effort(self) -> bool: + return False + def post_stream_processing(self, stream: object) -> object: """Hook for providers to post-process streaming responses. Default: pass-through.""" return stream diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index bed7bc1bd74..257fafbb6b8 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -27,6 +27,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO, ) from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator from litellm.litellm_core_utils.prompt_templates.factory import ( encode_tool_call_id_with_signature, @@ -926,11 +927,38 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # - Requires propertyOrdering optional_params["response_schema"] = self._map_response_schema(value=schema) + @property + def accepts_reasoning_summary_in_reasoning_effort(self) -> bool: + return True + + @staticmethod + def _include_thoughts(requested: bool | None) -> bool: + if requested is not None: + return requested + return normalize_drop_params(litellm.gemini_include_thoughts_default) is not False + + @staticmethod + def _include_thoughts_requested_by_reasoning_effort(reasoning_effort: object) -> bool | None: + match reasoning_effort: + case {"summary": str()}: + return True + case _: + return None + + @staticmethod + def _map_reasoning_effort(effort: str, model: str, reasoning_effort: object) -> GeminiThinkingConfig: + include_thoughts: Final = VertexGeminiConfig._include_thoughts_requested_by_reasoning_effort(reasoning_effort) + if VertexGeminiConfig._is_gemini_3_or_newer(model): + return VertexGeminiConfig._map_reasoning_effort_to_thinking_level(effort, model, include_thoughts) + return VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(effort, model, include_thoughts) + @staticmethod def _map_reasoning_effort_to_thinking_budget( reasoning_effort: str, model: str | None = None, + include_thoughts: bool | None = None, ) -> GeminiThinkingConfig: + sends_thoughts: Final = VertexGeminiConfig._include_thoughts(include_thoughts) if reasoning_effort == "minimal": # Use model-specific minimum thinking budget or fallback # Check for exact matches first, then partial matches @@ -945,22 +973,22 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return { "thinkingBudget": budget, - "includeThoughts": True, + "includeThoughts": sends_thoughts, } elif reasoning_effort == "low": return { "thinkingBudget": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, - "includeThoughts": True, + "includeThoughts": sends_thoughts, } elif reasoning_effort == "medium": return { "thinkingBudget": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - "includeThoughts": True, + "includeThoughts": sends_thoughts, } elif reasoning_effort == "high": return { "thinkingBudget": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, - "includeThoughts": True, + "includeThoughts": sends_thoughts, } elif reasoning_effort == "disable": return { @@ -987,6 +1015,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): def _map_reasoning_effort_to_thinking_level( reasoning_effort: str, model: str | None = None, + include_thoughts: bool | None = None, ) -> GeminiThinkingConfig: """ Map reasoning_effort to thinking_level for Gemini 3+ models. @@ -998,17 +1027,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): GeminiThinkingConfig with thinkingLevel and includeThoughts """ supports_minimal: Final = bool(model) and VertexGeminiConfig._supports_minimal_thinking_level(model) + sends_thoughts: Final = VertexGeminiConfig._include_thoughts(include_thoughts) if reasoning_effort == "minimal": if supports_minimal: - return {"thinkingLevel": "minimal", "includeThoughts": True} + return {"thinkingLevel": "minimal", "includeThoughts": sends_thoughts} else: - return {"thinkingLevel": "low", "includeThoughts": True} + return {"thinkingLevel": "low", "includeThoughts": sends_thoughts} elif reasoning_effort == "low": - return {"thinkingLevel": "low", "includeThoughts": True} + return {"thinkingLevel": "low", "includeThoughts": sends_thoughts} elif reasoning_effort == "medium": - return {"thinkingLevel": "medium", "includeThoughts": True} + return {"thinkingLevel": "medium", "includeThoughts": sends_thoughts} elif reasoning_effort == "high": - return {"thinkingLevel": "high", "includeThoughts": True} + return {"thinkingLevel": "high", "includeThoughts": sends_thoughts} elif reasoning_effort in ("disable", "none"): return { "thinkingLevel": "minimal" if supports_minimal else "low", @@ -1061,6 +1091,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): status_code=400, ) + @staticmethod + def _thinking_asks_for_thoughts(thinking_param: Mapping[str, object]) -> bool: + display: Final = thinking_param.get("display") + return isinstance(thinking_param.get("summary"), str) or (display is not None and display != "omitted") + @staticmethod def _map_thinking_param( thinking_param: AnthropicThinkingParam, @@ -1068,6 +1103,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) -> GeminiThinkingConfig: thinking_enabled: Final = thinking_param.get("type") == "enabled" thinking_budget: Final = thinking_param.get("budget_tokens") + asks_for_thoughts: Final = VertexGeminiConfig._thinking_asks_for_thoughts(thinking_param) + sends_thoughts: Final = VertexGeminiConfig._include_thoughts(True if asks_for_thoughts else None) params: Final[GeminiThinkingConfig] = {} @@ -1077,7 +1114,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if thinking_budget == 0: params["includeThoughts"] = False else: - params["includeThoughts"] = True + params["includeThoughts"] = sends_thoughts # Follow provider defaults unless explicitly opted into legacy behavior. if litellm.enable_gemini_default_thinking_level_low is True: params["thinkingLevel"] = ( @@ -1089,7 +1126,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): else: # For older Gemini models, use thinkingBudget if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget): - params["includeThoughts"] = True + params["includeThoughts"] = sends_thoughts if thinking_budget is not None and isinstance(thinking_budget, int): params["thinkingBudget"] = thinking_budget @@ -1304,14 +1341,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): param_name="reasoning_effort", param_description="thinking_budget", ) - if VertexGeminiConfig._is_gemini_3_or_newer(model): - optional_params["thinkingConfig"] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level( - effort_value, model - ) - else: - optional_params["thinkingConfig"] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget( - effort_value, model - ) + optional_params["thinkingConfig"] = VertexGeminiConfig._map_reasoning_effort( + effort=effort_value, model=model, reasoning_effort=value + ) elif param == "thinking": # Validate no conflict with thinking_level VertexGeminiConfig._validate_thinking_config_conflicts( diff --git a/litellm/main.py b/litellm/main.py index d8e62ebcdad..e6caa5c25b6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -52,6 +52,7 @@ from litellm.utils import ( get_litellm_params, get_optional_params, peek_reasoning_summary_aliases, + reasoning_effort_with_summary_alias, strip_reasoning_summary_aliases_from_optional_params, ) @@ -5681,7 +5682,15 @@ def completion( "allowed_openai_params": allowed_openai_params, "base_model": base_model, } - optional_params = get_optional_params(**optional_param_args, **non_default_params) + provider_reasoning_effort: Final = reasoning_effort_with_summary_alias( + reasoning_effort, + non_default_params, + accepts_summary=provider_config is not None + and provider_config.accepts_reasoning_summary_in_reasoning_effort, + ) + optional_params = get_optional_params( + **{**optional_param_args, "reasoning_effort": provider_reasoning_effort}, **non_default_params + ) processed_non_default_params: Final = pre_process_non_default_params( model=model, passed_params=optional_param_args, diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index c2b4c105199..2552ee8c880 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -4,6 +4,7 @@ from urllib.parse import urlsplit import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, foundry_chat_rejects_function_tools_while_reasoning, @@ -11,7 +12,8 @@ from litellm.llms.azure_ai.common_utils import ( from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import OpenAIWebSearchOptions -from litellm.utils import get_model_info_helper +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, get_model_info_helper _OPENAI_DEFAULT_API_BASE: Final = "https://api.openai.com/v1" _OPENAI_API_HOST: Final = "api.openai.com" @@ -36,6 +38,18 @@ def _resolve_openai_api_base(api_base: str | None) -> str: ) +def chat_target_accepts_reasoning_summary(model: str, custom_llm_provider: str | None) -> bool: + try: + local_model, provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + chat_config: Final = ProviderConfigManager.get_provider_chat_config( + model=local_model, provider=LlmProviders(provider) + ) + except Exception as e: # noqa: BLE001 # a capability probe must never fail the request it probes for + verbose_logger.debug("chat target reasoning summary check failed: %s", e) + return False + return chat_config is not None and chat_config.accepts_reasoning_summary_in_reasoning_effort + + def responses_api_bridge_check( model: str, custom_llm_provider: str, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 7f5d143e98e..2e7fb6da194 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -56,6 +56,7 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control from litellm.llms.snowflake.chat.transformation import is_snowflake_claude_model +from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary from litellm.responses.litellm_completion_transformation.session_handler import ( ResponsesSessionHandler, ) @@ -785,8 +786,8 @@ class LiteLLMCompletionResponsesConfig: provider either rejects it or silently drops it, and dropping it turns reasoning off while still billing for the turn. ``summary`` has no chat equivalent, so it rides the ``reasoning_summary`` alias, which ``litellm.completion`` reassembles into ``{effort, - summary}`` when it bridges the model back onto the Responses API, and is sent to nothing - else. + summary}`` when it bridges the model back onto the Responses API or targets a chat provider + whose config maps a summary ask (Gemini's thought summaries), and is sent to nothing else. """ if not reasoning_param: return ResponsesReasoningChatForm(effort=None, summary=None) @@ -807,7 +808,10 @@ class LiteLLMCompletionResponsesConfig: reasoning_summary=summary, api_base=api_base, ) - return ResponsesReasoningChatForm(effort=effort, summary=summary if bridges_back else None) + forwards_summary: Final = bridges_back or ( + isinstance(effort, str) and chat_target_accepts_reasoning_summary(model, custom_llm_provider) + ) + return ResponsesReasoningChatForm(effort=effort, summary=summary if forwards_summary else None) @staticmethod def transform_responses_api_request_to_chat_completion_request( diff --git a/litellm/utils.py b/litellm/utils.py index c0c80f09e4a..65408663f6f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -89,6 +89,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, match_fill_missing_generalizations, ) +from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.litellm_core_utils.tokenizer import strip_special_tokens from litellm.rust_bridge import tokenizer as tokenizer_dispatch @@ -10446,22 +10447,21 @@ def get_non_default_completion_params(kwargs: Mapping[str, object]) -> dict[str, return filter_out_litellm_params(kwargs, excluding=litellm.OPENAI_CHAT_COMPLETION_PARAMS) -def peek_reasoning_summary_aliases(optional_params: dict) -> object | None: - """Read AI-SDK-style reasoning summary from optional_params or nested extra_body. - - Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped. - """ - if "reasoningSummary" in optional_params: - return optional_params["reasoningSummary"] - if "reasoning_summary" in optional_params: - return optional_params["reasoning_summary"] - extra_body: Final = optional_params.get("extra_body") - if isinstance(extra_body, dict): - if "reasoningSummary" in extra_body: - return extra_body["reasoningSummary"] - if "reasoning_summary" in extra_body: - return extra_body["reasoning_summary"] - return None +def reasoning_effort_with_summary_alias( + reasoning_effort: object, params: Mapping[str, object], accepts_summary: bool +) -> object: + summary: Final = peek_reasoning_summary_aliases(params) + if not accepts_summary or not isinstance(summary, str): + return reasoning_effort + match reasoning_effort: + case str(): + return {"effort": reasoning_effort, "summary": summary} + case {"summary": _}: + return reasoning_effort + case {"effort": str() as effort}: + return {"effort": effort, "summary": summary} + case _: + return reasoning_effort def strip_reasoning_summary_aliases_from_optional_params( diff --git a/tests/integration/translation/chat_completions/reasoning/__init__.py b/tests/integration/translation/chat_completions/reasoning/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/chat_completions/reasoning/test_chat_completions_reasoning_gemini.py b/tests/integration/translation/chat_completions/reasoning/test_chat_completions_reasoning_gemini.py new file mode 100644 index 00000000000..194e6972453 --- /dev/null +++ b/tests/integration/translation/chat_completions/reasoning/test_chat_completions_reasoning_gemini.py @@ -0,0 +1,1041 @@ +import json +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from pathlib import Path +from typing import Final +from unittest.mock import ANY +from uuid import uuid4 + +import pytest +from integration._support.client import Gateway, eventually, list_value, object_value, string_value +from integration._support.process import graceful_stop_seconds +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from integration.caching.response_cache_case import CACHE_KEY, redis_store +from integration.translation.case import SSEFrame, TranslationTestCase +from integration.translation.chat_completions.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE +from integration.translation.chat_completions.bases.vertex_ai import ( + GEMINI_3_8_FLASH_TEST_CASE as VERTEX_GEMINI_3_8_FLASH_TEST_CASE, +) +from integration.translation.gemini_thoughts import ( + LONG_SUMMARY, + gemini_thoughts_proxy, + sent_thinking_config, + thoughts_off, + with_thinking_config, +) +from integration.translation.messages.bases.gemini import ( + GEMINI_3_8_FLASH_TEST_CASE as MESSAGES_GEMINI_3_8_FLASH_TEST_CASE, +) +from integration.translation.responses.bases.gemini import ( + GEMINI_3_8_FLASH_TEST_CASE as RESPONSES_GEMINI_3_8_FLASH_TEST_CASE, +) +from integration.translation.runner import ( + assert_provider_request, + assert_translation, + encode_sse_frame, + parse_sse_frames, +) +from pydantic import JsonValue + +__all__ = ("thoughts_off",) + +pytestmark = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +GEMINI_3_8_FLASH_THINKING_SHOWN_THOUGHT: Final = "**My Inspection Strategy: Validation First**\n\nOkay, here's how I'm approaching this project review. The instruction specifically mentions considering zero, negative, and non-integer quantities. To efficiently address that, I'm going to start with `shop/validation.py`. My reasoning is simple: the validation logic is the first line of defense. Understanding how the system *expects* to handle these edge cases will inform everything else. It's the logical place to begin my investigation to understand the code. I'll use the `read_file` tool to examine that file. I am **not** changing any files. I will call the `chat_completions` client for this project review. I am now proceeding with reading `shop/validation.py`.\n\n\n" + +GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE: Final = ( + "EowMCokMAWkUfRPuOEs7DW4IXhoH/SNh9qYezwNSqhXT2tKoI1mcXDG+yx7R+12sMmzvjHh2OSghAlsvmR6welOzIM1LMw6c" + "QrSX2IPRo/xbtUvDLlo6aqufzRtini5EF0yxQdmyn9r9xznzDyuEf4eqrbK0KQAArEVM13rm3CTDIjyDRQPawLALfUh/e61/" + "XVcWrr1PZLR5Jj46ImyrobGGLmJuoytm3ezTEihTb2b0q3/vHOW9cO/gJzMcbCI8koexQLuBB1KLSHUCaZZQYPj4jLx/g0de" + "2l3H/DYR7Cdroh0P/R3mewkhu+3D2ELnHKt85InrdQOqdXp/XhICsivPgsIEWHMRr6kQ3ZLIUJvJbJ7zo7/4bYnHOm+JqmoX" + "E+VMobI2YZ+ff9GOYg4vsJRTeF7ff9kZHYBWRM9XvTsZMbeCZHRbCOLNmSSJPfn2ToDOgMxU1u/GqHYmVSV97ybT3Y0mxwtf" + "2WcVlPUCkFX0GW3GP5IHVkEISo0TznJSI3/yYb+5UmQFMuuIedDdhKa74eU1/K59VuTnd3DzVX/lfhpFbGLDkOSurRsDsViN" + "69zq+6VbLi+5ARi6SJqUU+1U02QaSIQVp8kxtrhEIsPedjCIYHQQNxR8hFPMhnEssR51THNELWH6sVccxpJIbFfGjl0TZS9d" + "rfYL2c+IBX56lrH8IX2gz1iMOsESKwTyAECmqJIL7d/HcVdTk404Hd9SYycB+2MP2GN2XA7ivoO/zfkkFcf+PGmXPed8dHlW" + "bAxrQcOAz89p8kPzb1bou5QUT3feeRG06aJL31gX7BSYj19mvLeo0AmMzx+Wb91NFlwUvlJYlhJo4NfZWg/ymcGclgM8r7iE" + "MmWZa8wul8g4hbJOg3GE9NV70zvKqEpQrJLTa9jHca0DBvj8N3C+XvpeEg+Dys0cxE7Ak0VVYbvu9r0OHKSWtC7NDYrJsF2n" + "VFmdavSxpyPEqPaAdWdiewDH0nhP5GCR/ciikokWFQMF5iClWRfL72qEbFx3cQkctqEf+kK2CevIWKPf8bvkrKVDdPqbjk0G" + "9NWuud+YgNxl4dSCkAWoMrAFlxXozEqMmanMAtZk/7nPeOVFbYEL/xOpSCfxaEKS1zEmMwTb/UNVAogsNY9YdBSFYVNSrS8F" + "N/IrHBvsP66+YEFuR46Di6cWuuZQdSOfB2uQe1eSC8IgrQ6XUhQHWeXKUXfbcb/1abPOi5PUMYyLBQ7r52UKCEfOBc1j0oNL" + "wYzI/vWAz74gq8NblQhHB2VlrInJ9rRVY1MehInqUH0kcG6ysQDQLiLJiS30TV0bZh05I86RfcjaN1twbRir82YchRjXG8/1" + "6iFRJ54Sk7oVzbdnbFOugDea8qIblFmgUTq0+SRLGBFcDMvAqYLNcR2ZO7o0xinMG7IuNRwoSMahnmreWYr9saVJAScd3All" + "i6RQuxjGIdxG0s0UNuoWiSNIf1Z2ngoJrvZHTc0iZK+A2zv3CBqWAaS8WGwSfcItnlc6hMB0bTWasoCelmRjfv+1jQnKzKMj" + "DzZlO6skMkv5x9GzLrAvpYoulLYa/wp9+exw/Ll7MxSB22aeAGfai9C7Vj310y8nApsG3QxaoUUO4jg1eCjrrluBLgoVY8cE" + "/O95IAUa2rFzVrOWbHuob7TEI/8+BwieGeyeHdPGn4fNeKYMkRjFeVJ1GA/2+iCHCjCbSNm4sdzh9188/Es3q4acRfQUiqwR" + "iQzW2E1fy9mXRDY9y5fdXuzjHtuCkH8MqR0cioN1p4OGZzPi6xXFaqV576NLjySBpfpQEYeXRA3j5NVwTwbeYKnGOot4bEtJ" + "JQfelwkXLXeAfxzCTD0/vGtdhilzEK8wSYM9a+8llr5D4XwtEsu/68YbXJ2jwzsS3rE/Frulrf6paz9fS45ZXj98+OT5EpFO" + "o5XHfTXKb5Brzr0tKfPSB2Mx6fXbYP3lZRwfJIaUq8Alwdo7g2jnHQUhCNBB8hkb/IkQrnhsAGS6f0ZEjwf6ErVOa8DkfizN" + "y4WTdS1D3gOUJxOcszm8J5b085tsfFxEJVgA0VjI60DBY0owWn2I" +) + +GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "To understand how zero, negative, and non-integer quantities (such as floats or invalid types) are currently handled and where the checkout rejection is enforced, we need to inspect `shop/validation.py` next.", + }, + { + "functionCall": { + "name": "read_file", + "args": { + "path": "shop/validation.py", + "end_line": 50, + "start_line": 1, + }, + "id": "call_330625", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 358, + "candidatesTokenCount": 77, + "totalTokenCount": 793, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 358, + }, + ], + "thoughtsTokenCount": 358, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "xy3RapaNEbT4-sAPhMrt6AU", +} + +GEMINI_3_8_FLASH_THINKING_OMITTED_THOUGHT_SIGNATURE: Final = ( + "EscECsQEAWkUfROlWYT6hX0DWOKSCg4MSuLzOV7u43EgJDn0dfS2R8v/MVGGcNolGA9rteb/cV3dyCSQ7D2EO2CB5whDkblj" + "fGAAw6NBhqnidYNQiXbsg7oqynxS116ZQnk302S4KzvAbIvo4gpRbahPVClXKkP8PCQWA5eRQ/e9oQeHr97gbBB7+PLsl0NK" + "m7YPV6lP3fAXx/r3EADbdNL1diHylKd6tCWUKRea/x2QIOBQZdY5jHSU73z/pnPOotVwggEiasDLSLA+uUR1KllgGc8WKZ4i" + "uRM3L1gDr1QZ4Rht3quiF8AwfopWE5m2ON+FD78aHUDG/lgDs0vXnJluzri9LpPaexfbpvwMPtAx8Bxrq1/dCeekR2VnVx1c" + "qZ7TSoWMqk4x98mPpyUFz1hSBRnXaOGvQl5Wk7Iart49tH2LoIBYoV+PkCgNZ2MOi2uzc7OlN/8gvOeBM05xbGAplsLiuKd4" + "jkgv7Nz8Jby3v/hxtQT9+Ow3mxwSaiIfEkuaxkPiK1cX1nwT92gFzpmhVsrGqpiCUdAfos39Wd2/rcrFbtl5iMAVcHG5JGjw" + "8AjubFqiWDMqWPC58FfsX9TLC2BMUbPf2xi3hrw+1QWDtSuNl9s+dofkCogXFdLA9lAFc0oRj3MVhzYUhn5eixApJdggDsJF" + "iJaQtRihs4dstbXX9QXYFnx22Ql3x9TWJibiH8jz4fJVOf0wreMWZ2xVXBiw2xKdy5dB7xRDnrQD/sdcpTpCWURYXVCXglJ9" + "DD7e7/OCNMLCPw==" +) + +GEMINI_3_8_FLASH_THINKING_OMITTED_RESPONSE: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "To understand how the checkout validates quantities\u2014including zero, negative, and non-integer quantities\u2014I need to inspect `shop/validation.py` next.", + }, + { + "functionCall": { + "name": "read_file", + "args": { + "start_line": 1, + "path": "shop/validation.py", + "end_line": 40, + }, + "id": "call_745953", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THINKING_OMITTED_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 358, + "candidatesTokenCount": 65, + "totalTokenCount": 531, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 358, + }, + ], + "thoughtsTokenCount": 108, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "zS3RasS-Lp2-jMcPoay98As", +} + +GEMINI_3_8_FLASH_THINKING_SHOWN_THOUGHT_SIGNATURE: Final = ( + "EqUECqIEAWkUfRNZEW6qD3+V/1LAc1riV0fvqkg4Bldd7YYDfln+ZHNg6Lbw2T31UZHTum0qYvrAOor848+zSkFG7qBDfdZ9" + "69FGl/dQGq2llr1yrv23blnbwC7tO0uevr8225EZ/4Pv+hbfP0teFcSZsasE4LQangtxYkewghc6cH1Acz293SIz1cqF/3CM" + "Ta+FtJ5x106oauWy79+z1VbV9TNyHpNtsrUy3ADR2PFNBjXKf2TvUWygzR4w2RJg581+dfx0sI97O3UlZrhSKRe+p/6siUP7" + "qnysURAia7k+ew5f05Nuo1aDwD2G20G2s9v7L0PxnCCAay5L4YTub5LXxvUGBXMhSHVb9MmO8+2rNPqDtPWh0bOQdKbkWAV/" + "dMckMLTKANDbnzBjtCK1HbN7Rw0sIBSnBVy59z7h+0ierHFgA8Vntfp/nvAy0wDpuZu09agdB3T6FO2HCvWCFG2Y+0B4lGJV" + "ErOknvn50gtn9Tpn1QSGv4VDRrrcv6nCOyohoL9CsBeO1Bs+RItyZSAoTA3bVB5IjcF2kKK27+/+VRJjN0cO4Cm5RD/ETb4+" + "kI101AFWJ3CYwKEu91SAVTtXLewQYd5LBiJ3Btt68m8d5uUp0a1kAn2cH6g2C7TOfoJ7BgU5+vU81mS8hk9QOFGb9RYUXs8Y" + "cJl21l6WfMMZri5bMI9qCAFn+YZdBBeIbGq+JJcIgjr/R2tULmU584+5beoEydkS" +) + +GEMINI_3_8_FLASH_THINKING_SHOWN_RESPONSE: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_3_8_FLASH_THINKING_SHOWN_THOUGHT, + "thought": True, + }, + { + "text": "I need to inspect `shop/validation.py` next to examine how zero, negative, and non-integer quantities are currently handled in `validate_quantity`.", + }, + { + "functionCall": { + "name": "read_file", + "args": { + "end_line": 40, + "start_line": 1, + "path": "shop/validation.py", + }, + "id": "call_386884", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THINKING_SHOWN_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 358, + "candidatesTokenCount": 66, + "totalTokenCount": 529, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 358, + }, + ], + "thoughtsTokenCount": 105, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "0C3RavXPIJWX9MoPtorosAs", +} + +GEMINI_2_5_FLASH_THOUGHTS_OFF_RESPONSE: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "I will read the `shop/validation.py` file to understand the quantity validation logic.", + }, + { + "functionCall": { + "name": "read_file", + "args": { + "start_line": 1, + "path": "shop/validation.py", + "end_line": 20, + }, + }, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 334, + "candidatesTokenCount": 53, + "totalTokenCount": 387, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 334, + }, + ], + "serviceTier": "standard", + }, + "modelVersion": "gemini-2.5-flash", + "responseId": "0y3Rar76LO_K-8YPh6Kd0Ak", +} + +VERTEX_GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE: Final = ( + "AY89a18W+yVUlI0b2XsXQujlNKztnSpKXMf0+1JFiprxfp7kc0bG2g9WoPITuNkCqkuBaO5SkZaKDrKQTlJFJKj+H6SnETLk" + "JrR3SAzp2GTjwM2QRsJjI5ky54oylVT93Jh51PWCMp6vl8tZstMJrois8b1i4/eTOyKaVprypLW8DOIImqN3Cbkea8nEDe9r" + "XsdpdvsR7i+NvD0eiCW+Z4ZIflx5NVENbyEezjyJCL3lg3bV2Mfb0ytsACHXuqgGQEcH9HBKQKGkF5n7mA0GNLw6wAqlk8FE" + "JKttP6NgS+HzS/U+xaqxci6B50l0hIMw3pTOHu8Gg5fRKxf/dzvks+s709/ODCGBDrhMzpf/ilnPAMrB67cQrHvQOhnI2pGk" + "cvkZv+ZGwAQxjJHs/r0aXDGU2mwN9RY+/rSEv8y4MWd5c1KIs7xa1GytDBaaG8Zwh3kZzaSOYjVKIk7cy31mPJ01jCNdW9Pg" + "1p2r2iwbzJrBHo5Ku3s1yK4rADeY1WOU63sM6evVkWJEtu6KPxAqE9JkQKQoAPOFtoP89RtiTSl0nZZaO0stbqG7G+AkE2iO" + "5WkKCB1o55R7EEZDalDgFPput8khce5hPrhIGikN3yo11sHsAA2PAuDcHo4L3eWaYPExFiZtB+gVTbBwx42ST3UuBB9lrHtk" + "GCrp7tlOBHJJY+RlXj1G0TJsSxUZDSEswCB+gjG+1tdEchKGRRKOBUjWGARCozBsIEuBOR0F1ugSCLYn7CumS2bLwmMhKY7+" + "mfKXEgF2U84MkmwDEOBSTubNbRhy0qNhVPcLT8pJvGvtTvLny1C90TBTHErVzOKb3Z095fFpUyW7MmFQj5J9rX3A160W0SNQ" + "RKXf2Yr6ikyVhLBSFNvQ00n81iXlyg7bp++ChbbCJBFCUcrGF5WnPLk3+isixodP+G0b+pNtCNo8xMkrvqGqL5AMlmGJ9Qm4" + "a9S6QgDRaw5AzcV/W90a7/BcaIeqkIly3Z9M0+d3Eep4igYj6AK0JzX1BhchMogq9g/dquTTZZk4edVKflBmQcwg4qGGvioI" + "X/ae4tKJVqB2U0Ed+A6t3JZ4Ggdpopdnuy29XvdEyTmUg+fF2sTCgAdK07hueKYO2pkhqj86mSy88iHV0/secWQs0ZuHtcSW" + "D9lAG57yvpMf6RIwccLTyVmM4jcO0A2OB0sTnNIVKYYtswPtUqbrPMTR5saBfEfnULipI521kpb3BseODBKar+G1FISLrb0Q" + "OjMhNnCWCoCSVDk1FbDmtEvSjfiUzeJtFYoqM8PIXyGiBZnsmjGaOw3QMNvLfkoy4gqE6nvFVgwIk3+J6Z1GdFecCPSSCatz" + "LRwwQ8B5ZdlJ6+xjTDWIuyaOByD5BsB98e0YHyuLSYP17w6+0pSGXf3M6AdBJXK7hcL/vu3sGzftYARFHgRY1PQsYCqW8RVy" + "uJL/GwEAT4EgzkDDWufizfUHJEAUVBpQKkxZW7eTbyHyPS/Uiu5a74f5L8lYyep1pAdqt3iK4lvmQ3T9h3DAQIQ15T8PLV78" + "FjzJvZDQ2uMryvoRS1MGEcK6dxc4VvcWalQTjyczE4jQIxEEiN3U5Yo5CaM6FFGDnjIJBiy0fUD16P8e4ZaEaRFQCY4QZ0sA" + "bhwSHWhnU7QWvA/CVnqOVni5TpRsY1ShtEaQXKTslUuz96lB6XwEeaDkPqsPdw+mWY9QmmLqXnXERnTq13rQY1s=" +) + +VERTEX_GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "To understand how zero, negative, and non-integer quantities are handled during checkout validation, we need to inspect `shop/validation.py` next.", + }, + { + "functionCall": { + "name": "read_file", + "args": { + "start_line": 1, + "path": "shop/validation.py", + "end_line": 50, + }, + "id": "call_1272842", + }, + "thoughtSignature": VERTEX_GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE, + }, + ], + }, + "finishReason": "STOP", + }, + ], + "usageMetadata": { + "promptTokenCount": 246, + "candidatesTokenCount": 64, + "totalTokenCount": 583, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 246, + }, + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 64, + }, + ], + "thoughtsTokenCount": 273, + }, + "modelVersion": "gemini-3.8-flash", + "createTime": "2026-10-15T20:02:49.484984Z", + "responseId": "aTHRavjMHYvi6tkPt6OroAo", +} + +GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_THOUGHT_SIGNATURE: Final = ( + "ErkLCrYLAWkUfRO3TXF/xF4wcqKCnAFmIW9MEkIMZeaFFG/BmstoEFBw+0v0jari9LOu/TkcHKNFvewXanY1T+6rm512Fsvm" + "3cbxU5C7oP8kGPeEHIBOs2BIeocBFx3ZXSmIn25pOLd2YOqDaTzu/De/QX2JUkno3r2Ozbvuy0wvS5EsD3816rYYnaYc7+4E" + "iZRQhf5r6notr70qcHJ3DuKIP8ghDEz0gNSN1bAUa8N5Hvjh7ITbHLoSkeKO0pOjXaSWfQ86xv9F2zhpMZfW+muxcm1+g9tT" + "5QkyJxMtn+cnHi/w4MbQy+9tbJWdkkLo2H6z81pBsmjvIZAQnjjF0zP8aL91/+S9Po8HNV0LsrGs4P4kUgRR33I6UPMCVnHf" + "B7IxmVSEIGeFQ8gg1mJRDj7yZJFBgf61ya97bVWAhiAm6t/EHdnMi9xIPFEYSrY6ed8ZGUnf05x/GRMVOfrmeCePZIHXpftk" + "UfI2W4+qIM1YGR8+GD7KOtQEkcsrjRGHvhmcv1GW3NlLVpZ4LWIqX6ZtmZrOjDFvgnCg/jYdzDHL5j2vQiiSlcpsHJ4UZWRj" + "NmqUyUY0OZC77erTqensEeWJggUfyizdp3InLZtWM/C5B6rhLGku3LgGwAxdSf5aY25i0f+Ayl/xE/nnCkXS81D64z/+Xqdn" + "W4jsZn2tg8TGb11T6GncBK82aE/T9zVUHT7mbUtEljdinxNc/NS+9mWEkhtLSOAQqQmGF24oxMFlCcejVhGZz5QnFpjnu2Uk" + "COv2AZ7S9iYJQa/xT1LyQ18IibvAUWQPRBpjwRnOf0wi0beuE9Q/h0Y2itFjCt+mz5JlFRyQRJcpEOupOsfF47ULZCLeG0Uz" + "Af8JQzHMUWL3ESlOs30cLQQBJwtHW3QD3ZA1uwMCZnJcpBvIr7GhRdF/VhKZ7C8aYZLMOk32M1i+iGe5lYOM2WuoEzbprTGz" + "aVQeQBWEIdyT+RhXjpvLrrsWOIWI/OAw1Y8VEtouLt246yDjDrVuKUdJfXPz9VfnJ/7dKmsYWAaOCBnQ6edZ9HY8vzEKyuma" + "mcaJWSrwuq1mIouQ1Pw4fvazi6JE2xx17hEuOpS5sLD8E89V9uUbkcDne4XFoHNF9mm3nVgTplNFqo5J4kSiEZJ6qJveiMyF" + "sw3F7SbuU0f95AE/HVwMfDhBbnT3j0R5s2zejxLW6mWuASodcn+J9WGbRfMDLZV+aozCPy+rrXrIfeVQw8nLfjQbwY/iQLFM" + "rQCM4m9F5MLrIM0jGKePUeLRmkhJ92LiqEmDMkOZct09jVaZpSW6dRO+ANGfxqJ0y8KppBA7jgWy/PDKMHfwQFWh3Goxc86m" + "otxTUbF2i15mhhPGN2Z5xsRe1lsYRsFV7LNIkqLXcUVQ7uQGmlzi7wCGiMasvxDgzTLdTGAJ6FsaWzr+GAHH/RFSyRA9a6Qr" + "/Z/h9tm5sx5l6hTQrYspK3qK+cYI7BJCuiYYiIMWEAV6qb5SNhMZ88UlTayryuoolQ7pBco59MBV+P6oAoWAXFh69D4yDAe8" + "L2iNa6UTLt2BSMpAgqE5HdrsifGPfwOwNrwtfv/t4eXIBpBVgqC4csZSrN4zUsP6fH3mPZ4oPgwkDFaLu/x1S/PdxKvF0mYg" + "e3ORBogDHyO00QzHPP/9sl2PEgzIQmQv+0tkmpmAeTUfU8FMlRouFsgbrO2wUx1P8KoaFO4ZToQHeNaoqhI2yZ+GB9EquuBp" + "Xpu46qaU1Wrf3w7vtI0aLUPDKnudfZ3b/me3LVm1cgutomL5o33J86U0v62TwJz8xnLB0881oHQVXX4VfaPV4BVk74QCMcqK" + "/SRt3GpK9V1OGQh47ubs5MUFBnqauOnOKmttTxerI5QNi/t5vSwIvdy2MBcXnCXPJSjt3MpZjSZo34PQHK+1iuR1BCbmmHQk" + "v2UCtcYUZfnJ4FuKNXaZ/AFDT7VxbdnOaS8HVA==" +) + +GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_FRAMES: Final = ( + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "To understand how zero, negative, and non-integer quantities are handled during checkout, I need to inspect `shop/validation.py", + }, + ], + "role": "model", + }, + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 358, + "candidatesTokenCount": 27, + "totalTokenCount": 673, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 358, + }, + ], + "thoughtsTokenCount": 288, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "yi3RaufbH6bXjMcPt5mF2Q0", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "` next.", + }, + ], + "role": "model", + }, + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 358, + "candidatesTokenCount": 30, + "totalTokenCount": 676, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 358, + }, + ], + "thoughtsTokenCount": 288, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "yi3RaufbH6bXjMcPt5mF2Q0", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "read_file", + "args": { + "end_line": 50, + "path": "shop/validation.py", + "start_line": 1, + }, + "id": "call_317677", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 358, + "candidatesTokenCount": 63, + "totalTokenCount": 709, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 358, + }, + ], + "thoughtsTokenCount": 288, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "yi3RaufbH6bXjMcPt5mF2Q0", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "", + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 358, + "candidatesTokenCount": 63, + "totalTokenCount": 709, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 358, + }, + ], + "thoughtsTokenCount": 288, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "yi3RaufbH6bXjMcPt5mF2Q0", + } + ), +) + +GEMINI_2_5_FLASH_REQUEST: Final = with_thinking_config( + { + **GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, + "contents": [ + list_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["contents"])[0], + { + "role": "model", + "parts": [ + {"text": "I will locate the quantity validation and its tests."}, + { + "function_call": { + "name": "search_files", + "args": {"query": "quantity", "paths": ["shop/", "tests/"]}, + } + }, + ], + }, + { + "role": "user", + "parts": [ + { + "function_response": { + "name": "search_files", + "response": { + "matches": [ + { + "path": "shop/validation.py", + "line": 18, + "text": "def validate_quantity(quantity):", + }, + { + "path": "tests/test_validation.py", + "line": 7, + "text": "def test_zero_quantity():", + }, + ], + }, + }, + }, + ], + }, + list_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["contents"])[3], + ], + }, + {"thinkingBudget": 4096, "includeThoughts": False}, +) + + +def gemini_chat_response( + *, + content: str, + tool_call: Mapping[str, JsonValue], + message_fields: Mapping[str, JsonValue], + usage: Mapping[str, JsonValue], +) -> Mapping[str, JsonValue]: + return { + **GEMINI_3_8_FLASH_TEST_CASE.expected_litellm_response, + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": content, + "role": "assistant", + "tool_calls": [dict(tool_call)], + "images": [], + **message_fields, + }, + "provider_specific_fields": {"native_finish_reason": "STOP"}, + }, + ], + "usage": dict(usage), + } + + +def signed_tool_call(call_id: str, arguments: str, signature: str) -> Mapping[str, JsonValue]: + return { + "index": 0, + "provider_specific_fields": {"thought_signature": signature}, + "function": {"arguments": arguments, "name": "read_file"}, + "id": f"{call_id}__thought__{signature}", + "type": "function", + } + + +THINKING_REQUEST: Final[Mapping[str, JsonValue]] = { + key: value for key, value in GEMINI_3_8_FLASH_TEST_CASE.litellm_request.items() if key != "reasoning_effort" +} + +GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE: Final = replace( + GEMINI_3_8_FLASH_TEST_CASE, + scenario="thoughts_off", + expected_provider_request=with_thinking_config( + GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, {"thinkingLevel": "high", "includeThoughts": False} + ), + mock_provider_response=GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE, + expected_litellm_response=gemini_chat_response( + content=( + "To understand how zero, negative, and non-integer quantities (such as floats or invalid types) are " + "currently handled and where the checkout rejection is enforced, we need to inspect " + "`shop/validation.py` next." + ), + tool_call=signed_tool_call( + "call_330625", + '{"path": "shop/validation.py", "end_line": 50, "start_line": 1}', + GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE, + ), + message_fields={ + "thinking_blocks": [], + "provider_specific_fields": {"thought_signatures": [GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE]}, + }, + usage={ + "completion_tokens": 435, + "prompt_tokens": 358, + "total_tokens": 793, + "completion_tokens_details": {"reasoning_tokens": 358, "text_tokens": 77}, + "prompt_tokens_details": {"text_tokens": 358}, + }, + ), +) + +GEMINI_2_5_FLASH_THOUGHTS_OFF_TEST_CASE: Final = replace( + GEMINI_3_8_FLASH_TEST_CASE, + scenario="thoughts_off", + litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "model": "gemini/gemini-2.5-flash"}, + expected_provider_endpoint="/models/gemini-2.5-flash:generateContent", + expected_provider_request=GEMINI_2_5_FLASH_REQUEST, + mock_provider_response=GEMINI_2_5_FLASH_THOUGHTS_OFF_RESPONSE, + expected_litellm_response={ + **gemini_chat_response( + content="I will read the `shop/validation.py` file to understand the quantity validation logic.", + tool_call={ + "index": 0, + "function": { + "arguments": '{"start_line": 1, "path": "shop/validation.py", "end_line": 20}', + "name": "read_file", + }, + "id": ANY, + "type": "function", + }, + message_fields={"thinking_blocks": []}, + usage={ + "completion_tokens": 53, + "prompt_tokens": 334, + "total_tokens": 387, + "completion_tokens_details": {"text_tokens": 53}, + "prompt_tokens_details": {"text_tokens": 334}, + }, + ), + "model": "gemini/gemini-2.5-flash", + }, +) + +VERTEX_GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE: Final = replace( + VERTEX_GEMINI_3_8_FLASH_TEST_CASE, + scenario="thoughts_off", + expected_provider_request=with_thinking_config( + VERTEX_GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, {"thinkingLevel": "high", "includeThoughts": False} + ), + mock_provider_response=VERTEX_GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE, + expected_litellm_response={ + **gemini_chat_response( + content=( + "To understand how zero, negative, and non-integer quantities are handled during checkout validation, " + "we need to inspect `shop/validation.py` next." + ), + tool_call=signed_tool_call( + "call_1272842", + '{"start_line": 1, "path": "shop/validation.py", "end_line": 50}', + VERTEX_GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE, + ), + message_fields={ + "thinking_blocks": [], + "provider_specific_fields": { + "thought_signatures": [VERTEX_GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE] + }, + }, + usage={ + "completion_tokens": 337, + "prompt_tokens": 246, + "total_tokens": 583, + "completion_tokens_details": {"reasoning_tokens": 273, "text_tokens": 64}, + "prompt_tokens_details": {"text_tokens": 246}, + }, + ), + "model": "vertex_ai/gemini-3.8-flash", + }, +) + +GEMINI_3_8_FLASH_THINKING_OMITTED_TEST_CASE: Final = replace( + GEMINI_3_8_FLASH_TEST_CASE, + scenario="thinking_omitted", + litellm_request={ + **THINKING_REQUEST, + "thinking": {"type": "enabled", "budget_tokens": 1024, "display": "omitted"}, + }, + expected_provider_request=with_thinking_config( + GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, {"includeThoughts": False} + ), + mock_provider_response=GEMINI_3_8_FLASH_THINKING_OMITTED_RESPONSE, + expected_litellm_response=gemini_chat_response( + content=( + "To understand how the checkout validates quantities—including zero, negative, and non-integer " + "quantities—I need to inspect `shop/validation.py` next." + ), + tool_call=signed_tool_call( + "call_745953", + '{"start_line": 1, "path": "shop/validation.py", "end_line": 40}', + GEMINI_3_8_FLASH_THINKING_OMITTED_THOUGHT_SIGNATURE, + ), + message_fields={ + "thinking_blocks": [], + "provider_specific_fields": {"thought_signatures": [GEMINI_3_8_FLASH_THINKING_OMITTED_THOUGHT_SIGNATURE]}, + }, + usage={ + "completion_tokens": 173, + "prompt_tokens": 358, + "total_tokens": 531, + "completion_tokens_details": {"reasoning_tokens": 108, "text_tokens": 65}, + "prompt_tokens_details": {"text_tokens": 358}, + }, + ), +) + +GEMINI_3_8_FLASH_THINKING_SHOWN_TEST_CASE: Final = replace( + GEMINI_3_8_FLASH_TEST_CASE, + scenario="thinking_shown", + litellm_request={ + **THINKING_REQUEST, + "thinking": {"type": "enabled", "budget_tokens": 1024, "display": "summarized"}, + }, + expected_provider_request=with_thinking_config( + GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, {"includeThoughts": True} + ), + mock_provider_response=GEMINI_3_8_FLASH_THINKING_SHOWN_RESPONSE, + expected_litellm_response=gemini_chat_response( + content=( + "I need to inspect `shop/validation.py` next to examine how zero, negative, and non-integer quantities " + "are currently handled in `validate_quantity`." + ), + tool_call=signed_tool_call( + "call_386884", + '{"end_line": 40, "start_line": 1, "path": "shop/validation.py"}', + GEMINI_3_8_FLASH_THINKING_SHOWN_THOUGHT_SIGNATURE, + ), + message_fields={ + "reasoning_content": GEMINI_3_8_FLASH_THINKING_SHOWN_THOUGHT, + "thinking_blocks": [{"type": "thinking", "thinking": GEMINI_3_8_FLASH_THINKING_SHOWN_THOUGHT}], + "provider_specific_fields": {"thought_signatures": [GEMINI_3_8_FLASH_THINKING_SHOWN_THOUGHT_SIGNATURE]}, + }, + usage={ + "completion_tokens": 171, + "prompt_tokens": 358, + "total_tokens": 529, + "completion_tokens_details": {"reasoning_tokens": 105, "text_tokens": 66}, + "prompt_tokens_details": {"text_tokens": 358}, + }, + ), +) + + +def with_request(case: TranslationTestCase, scenario: str, **fields: JsonValue) -> TranslationTestCase: + return replace(case, scenario=scenario, litellm_request={**case.litellm_request, **fields}) + + +def with_thinking(case: TranslationTestCase, scenario: str, display: JsonValue) -> TranslationTestCase: + return with_request(case, scenario, thinking={"type": "enabled", "budget_tokens": 1024, "display": display}) + + +def with_summary(case: TranslationTestCase, scenario: str, summary: JsonValue) -> TranslationTestCase: + return with_request(case, scenario, reasoning_effort={"effort": "high", "summary": summary}) + + +@pytest.mark.parametrize( + "case", + ( + GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, + GEMINI_2_5_FLASH_THOUGHTS_OFF_TEST_CASE, + VERTEX_GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, + ), + ids=( + GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE.id, + GEMINI_2_5_FLASH_THOUGHTS_OFF_TEST_CASE.id, + VERTEX_GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE.id, + ), +) +def test_gemini_keeps_thinking_and_billing_reasoning_without_thought_text_when_the_gateway_turns_it_off( + case: TranslationTestCase, thoughts_off: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, thoughts_off, provider) + + +SUMMARY_ASKS: Final = ( + with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_auto", "auto"), + with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_empty", ""), + with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_long", LONG_SUMMARY), + with_summary(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_int", 7), + with_summary(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_list", ["auto"]), + with_summary(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_null", None), + with_request(GEMINI_3_8_FLASH_TEST_CASE, "summary_alias", reasoning_summary="auto"), + with_request(GEMINI_3_8_FLASH_TEST_CASE, "summary_alias_camel", reasoningSummary="detailed"), + with_request( + GEMINI_3_8_FLASH_TEST_CASE, + "summary_alias_effort_dict", + reasoning_effort={"effort": "high"}, + reasoning_summary="auto", + ), + with_request(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_alias_null", reasoning_summary=None), + with_request( + GEMINI_3_8_FLASH_THINKING_SHOWN_TEST_CASE, + "thinking_summary", + thinking={"type": "enabled", "budget_tokens": 1024, "summary": "auto"}, + ), +) + + +@pytest.mark.parametrize("case", SUMMARY_ASKS, ids=tuple(case.id for case in SUMMARY_ASKS)) +def test_only_a_string_reasoning_summary_brings_thought_text_back_over_the_gateway_default( + case: TranslationTestCase, thoughts_off: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, thoughts_off, provider) + + +def without_closed_schema(tool: JsonValue) -> JsonValue: + function: Final = object_value(object_value(tool)["function"]) + parameters: Final = object_value(function["parameters"]) + open_parameters: Final = {name: value for name, value in parameters.items() if name != "additionalProperties"} + return {**object_value(tool), "function": {**function, "parameters": open_parameters}} + + +def cacheable_run(case: TranslationTestCase, marker: str) -> TranslationTestCase: + messages: Final = list_value(case.litellm_request["messages"]) + system: Final = object_value(messages[0]) + instruction: Final = object_value(case.expected_provider_request["system_instruction"]) + (part,) = list_value(instruction["parts"]) + text: Final = f"{string_value(system['content'])} Review run {marker}." + return replace( + case, + litellm_request={ + **{name: value for name, value in case.litellm_request.items() if name != "cache"}, + "messages": [{**system, "content": text}, messages[1]], + "tools": [without_closed_schema(tool) for tool in list_value(case.litellm_request["tools"])], + }, + expected_provider_request={ + **case.expected_provider_request, + "contents": [list_value(case.expected_provider_request["contents"])[0]], + "system_instruction": {**instruction, "parts": [{**object_value(part), "text": text}]}, + }, + ) + + +def response_cache_keys() -> frozenset[str]: + keys: Final = (raw.decode() for raw in redis_store().scan_iter()) + return frozenset(key for key in keys if CACHE_KEY.match(key)) + + +def stored_reply_key(before: frozenset[str]) -> str: + (key,) = eventually(lambda: response_cache_keys() - before, lambda added: len(added) == 1) + return key + + +GEMINI_3_8_FLASH_SUMMARY_ALIAS_TEST_CASE: Final = with_request( + GEMINI_3_8_FLASH_TEST_CASE, "summary_alias", reasoning_summary="auto" +) + + +@pytest.mark.parametrize( + ("first", "second"), + ( + (GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, GEMINI_3_8_FLASH_SUMMARY_ALIAS_TEST_CASE), + (GEMINI_3_8_FLASH_SUMMARY_ALIAS_TEST_CASE, GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE), + ), + ids=("plain_then_summary_alias", "summary_alias_then_plain"), +) +def test_a_summary_alias_ask_and_a_plain_request_never_share_a_cached_reply( + first: TranslationTestCase, second: TranslationTestCase, thoughts_off: Gateway, provider: SharedProvider +) -> None: + marker: Final = uuid4().hex + before: Final = response_cache_keys() + assert_translation(cacheable_run(first, marker), thoughts_off, provider) + first_key: Final = stored_reply_key(before) + assert_translation(cacheable_run(second, marker), thoughts_off, provider) + second_key: Final = stored_reply_key(before | {first_key}) + repeat: Final = thoughts_off.request("POST", second.litellm_endpoint, cacheable_run(second, marker).litellm_request) + assert repeat.status_code == 200, repeat.text + assert repeat.headers.get("x-litellm-cache-key") == second_key, sorted(repeat.headers) + assert provider.received() == () + + +DISPLAY_ASKS: Final = ( + GEMINI_3_8_FLASH_THINKING_SHOWN_TEST_CASE, + with_thinking(GEMINI_3_8_FLASH_THINKING_SHOWN_TEST_CASE, "display_updates", "updates"), + with_thinking(GEMINI_3_8_FLASH_THINKING_SHOWN_TEST_CASE, "display_int", 7), + with_thinking(GEMINI_3_8_FLASH_THINKING_SHOWN_TEST_CASE, "display_empty", ""), + with_thinking(GEMINI_3_8_FLASH_THINKING_SHOWN_TEST_CASE, "display_list", ["omitted"]), + with_thinking(GEMINI_3_8_FLASH_THINKING_SHOWN_TEST_CASE, "display_long", LONG_SUMMARY), + GEMINI_3_8_FLASH_THINKING_OMITTED_TEST_CASE, +) + + +@pytest.mark.parametrize("case", DISPLAY_ASKS, ids=tuple(case.id for case in DISPLAY_ASKS)) +def test_any_thinking_display_but_omitted_brings_thought_text_back_over_the_gateway_default( + case: TranslationTestCase, thoughts_off: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, thoughts_off, provider) + + +DEFAULT_GATEWAY_DISPLAYS: Final = ( + with_thinking(GEMINI_3_8_FLASH_THINKING_SHOWN_TEST_CASE, "display_omitted_default", "omitted"), + GEMINI_3_8_FLASH_THINKING_SHOWN_TEST_CASE, +) + + +@pytest.mark.parametrize("case", DEFAULT_GATEWAY_DISPLAYS, ids=tuple(case.id for case in DEFAULT_GATEWAY_DISPLAYS)) +def test_thinking_display_never_turns_thought_text_off_on_the_default_gateway( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) + + +def test_a_streamed_turn_keeps_thinking_and_billing_reasoning_without_thought_text( + thoughts_off: Gateway, provider: SharedProvider +) -> None: + provider.expect( + Reply( + content_type="text/event-stream", + chunks=tuple(encode_sse_frame(frame) for frame in GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_FRAMES), + ) + ) + response: Final = thoughts_off.request( + "POST", + "/v1/chat/completions", + {**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "stream": True, "stream_options": {"include_usage": True}}, + ) + assert_provider_request( + provider.received(), + "/models/gemini-3.8-flash:streamGenerateContent?alt=sse", + GEMINI_3_8_FLASH_TEST_CASE.expected_provider_headers, + GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE.expected_provider_request, + ) + assert response.status_code == 200, response.text + frames: Final = tuple(frame.data for frame in parse_sse_frames(response.text) if frame.data != "[DONE]") + deltas: Final = tuple( + object_value(object_value(choice)["delta"]) + for frame in frames + for choice in list_value(frame.get("choices", [])) + ) + assert "".join(str(delta.get("content") or "") for delta in deltas) == ( + "To understand how zero, negative, and non-integer quantities are handled during checkout, I need to " + "inspect `shop/validation.py` next." + ) + assert [delta for delta in deltas if delta.get("reasoning_content") or delta.get("thinking_blocks")] == [] + usage: Final = [object_value(frame["usage"]) for frame in frames if frame.get("usage")][-1] + assert {field: value for field, value in usage.items() if field != "cost"} == { + "completion_tokens": 351, + "prompt_tokens": 358, + "total_tokens": 709, + "completion_tokens_details": {"reasoning_tokens": 288, "text_tokens": 63}, + "prompt_tokens_details": {"text_tokens": 358}, + } + assert isinstance(usage["cost"], float) and usage["cost"] > 0 + + +@pytest.mark.parametrize( + ("configured", "case"), + (("false", GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE), (None, GEMINI_3_8_FLASH_TEST_CASE)), + ids=("string-false", "null"), +) +def test_a_flag_written_as_text_or_left_null_still_sends_gemini_a_boolean( + configured: JsonValue, case: TranslationTestCase, gateway: Gateway, provider: SharedProvider, tmp_path: Path +) -> None: + with gemini_thoughts_proxy(gateway, tmp_path, configured) as owned: + assert_translation(case, owned.gateway, provider) + + +BURST_ASKS: Final[tuple[tuple[str, Mapping[str, JsonValue], Mapping[str, JsonValue]], ...]] = ( + ( + "/v1/chat/completions", + {**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "reasoning_effort": {"effort": "low", "summary": "auto"}}, + {"thinkingLevel": "low", "includeThoughts": True}, + ), + ( + "/v1/chat/completions", + GEMINI_3_8_FLASH_TEST_CASE.litellm_request, + {"thinkingLevel": "high", "includeThoughts": False}, + ), + ( + "/v1/messages", + MESSAGES_GEMINI_3_8_FLASH_TEST_CASE.litellm_request, + {"thinkingLevel": "high", "includeThoughts": True}, + ), + ( + "/v1/messages", + {**MESSAGES_GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "thinking": {"type": "adaptive"}}, + {"thinkingLevel": "high", "includeThoughts": False}, + ), + ( + "/v1/responses", + {**RESPONSES_GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "reasoning": {"effort": "high", "summary": "auto"}}, + {"thinkingLevel": "high", "includeThoughts": True}, + ), + ( + "/v1/responses", + RESPONSES_GEMINI_3_8_FLASH_TEST_CASE.litellm_request, + {"thinkingLevel": "high", "includeThoughts": False}, + ), +) + + +def _assert_each_request_follows_its_own_ask(gateway: Gateway, provider: SharedProvider) -> None: + burst: Final = BURST_ASKS * 4 + for _ in burst: + provider.expect(Reply(body=json.dumps(GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE).encode())) + with ThreadPoolExecutor(max_workers=12) as pool: + responses: Final = tuple(pool.map(lambda ask: gateway.request("POST", ask[0], ask[1]), burst)) + assert [response.status_code for response in responses] == [200] * len(burst), [ + response.text for response in responses if response.status_code != 200 + ] + sent: Final = sorted(json.dumps(sent_thinking_config(request), sort_keys=True) for request in provider.received()) + assert sent == sorted(json.dumps(expected, sort_keys=True) for _, _, expected in burst) + + +def test_every_worker_honors_the_flag_on_every_endpoint_under_a_burst_and_after_a_worker_respawns( + gateway: Gateway, provider: SharedProvider, tmp_path: Path +) -> None: + with gemini_thoughts_proxy(gateway, tmp_path, False, workers=2) as owned: + _assert_each_request_follows_its_own_ask(owned.gateway, provider) + owned.kill_worker(min(owned.worker_pids())) + _assert_each_request_follows_its_own_ask(owned.gateway, provider) diff --git a/tests/integration/translation/gemini_thoughts.py b/tests/integration/translation/gemini_thoughts.py new file mode 100644 index 00000000000..e0d89c3d43e --- /dev/null +++ b/tests/integration/translation/gemini_thoughts.py @@ -0,0 +1,63 @@ +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, list_value, object_value +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Request +from pydantic import JsonValue + +GEMINI_2_5_FLASH_DEPLOYMENT: Final[Mapping[str, JsonValue]] = { + "model_name": "gemini/gemini-2.5-flash", + "litellm_params": { + "model": "gemini/gemini-2.5-flash", + "api_base": "http://127.0.0.1:8191", + "api_key": "synthetic-gemini-key", + }, +} +LONG_SUMMARY: Final = "detailed " * 600 + + +def with_thinking_config( + request: Mapping[str, JsonValue], thinking_config: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + return { + **request, + "generationConfig": {**object_value(request["generationConfig"]), "thinkingConfig": dict(thinking_config)}, + } + + +def sent_thinking_config(request: Request) -> Mapping[str, JsonValue]: + body: Final = JSON_OBJECT.validate_json(request.body) + return object_value(object_value(body["generationConfig"])["thinkingConfig"]) + + +@contextmanager +def gemini_thoughts_proxy( + gateway: Gateway, directory: Path, default: JsonValue, *, workers: int = 1 +) -> Iterator[OwnedProxy]: + stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = directory / "gemini-thoughts.yaml" + config.write_text( + yaml.safe_dump( + { + **stock, + "model_list": [*list_value(stock["model_list"]), dict(GEMINI_2_5_FLASH_DEPLOYMENT)], + "litellm_settings": { + **object_value(stock["litellm_settings"]), + "gemini_include_thoughts_default": default, + }, + } + ) + ) + with owned_proxy_process(gateway, directory, {}, config=config, workers=workers) as owned: + yield owned + + +@pytest.fixture(scope="module") +def thoughts_off(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + with gemini_thoughts_proxy(session_gateway, tmp_path_factory.mktemp("gemini-thoughts-off"), False) as owned: + yield owned.gateway diff --git a/tests/integration/translation/messages/reasoning/test_messages_reasoning_gemini.py b/tests/integration/translation/messages/reasoning/test_messages_reasoning_gemini.py new file mode 100644 index 00000000000..c350d6e2a10 --- /dev/null +++ b/tests/integration/translation/messages/reasoning/test_messages_reasoning_gemini.py @@ -0,0 +1,373 @@ +from collections.abc import Mapping +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway, object_value +from integration._support.process import graceful_stop_seconds +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from integration.translation.case import SSEFrame, TranslationTestCase +from integration.translation.gemini_thoughts import LONG_SUMMARY, thoughts_off, with_thinking_config +from integration.translation.messages.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE, GEMINI_MESSAGES_RESPONSE +from integration.translation.runner import ( + assert_provider_request, + assert_translation, + encode_sse_frame, + parse_sse_frames, +) +from pydantic import JsonValue + +__all__ = ("thoughts_off",) + +pytestmark = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE: Final = ( + "EscLCsQLAWkUfRP4jtXWjNAugx5FeAwxasL/kKzH9gXUHHB2cTK13h4C8C6tKVYMKP5rXoPq4yhoXSrsDiIjUEpuAjdQYYCz" + "Wvx21tGAN4iBA4VELBqPOi92vo9YVGpczJ/XVREq9KNRFIkyMzWyEzR1lBbJFA4detKHzSVJ7VQ1iRBn9jk6xvknTCxQhbnl" + "Pct8TfMo4XluFsM2PY3qAqoX1DjkrjQNe4nrN9Yoglq8Bwr2ayyc3DUNkNJDu0DQN20W6HU7fbeqSizEOsvGIvJVEUZhMOFF" + "0r8NyzUGkNiiAZ+TEupIV+G0eTr6sVlMPLS1N74ryQ55sevdj6irenHlP19IwkiDNhy6ApRIoblY5I1uJVt4lj4tQyACGMAE" + "VOTg3Gb78hPgJ+8cF3ukmRF0H1bpYH3n/mpnkqiggfK3vm7fVnsCy03W7Keyl7xd7Gfto/HL7vPCta/sO6ZmZm28JvdCr/Ce" + "X/W82nHuEqcjiR8fNPk413rL/geE4Lppru1H9ESWBICXzwl6pLxaBBNr8BD/jGMvlYA/ZEcBfKWtYyzow3+yp+4M0fsY1Ah7" + "pxVCcCbivMRv2X5J5m9btU1IIm2REJicjMSWCxh4lQMEuuk2FaPzge+wXROC/Zx0HwNw2iPw6VpziTOByACQxXOE7RdWULq1" + "3rkbyXCMkuAfG281SiylOnJ3I3tkbRcFoTaY5DcMBKYCCmoJJZPO1HKJ2DMw5Spz1HDftQ7lg9df5FDoZbmehUZKfY+j/2LN" + "//Wr5OEUmuBpojC5ht3YYgXaxclo/pa70KnLHlQWHPZ3nqdB2ru0tqgOaXSv9KZNhtvXgagLsmffQ/rm5QiRoEeG4oZ5E6Ga" + "VLQB9OvWCDNdzgNVzss0tveNcG/cttJBnkSd5/azHSvzkCDQYh4J/GT3zDVvOyOayWTqzGSLgIoFGpTES7Vsbpph62apefvO" + "+ZAlxPNNo/3tHncHzGc8Pku07fF1yhiKdGRV1OjYndfQv6UuDc3G7n2oU90iCKWRoCqOYAotzw1yCh4lbsSpEWO2iB9kWW93" + "mJsdqQBdk6e9UiuHdJIOfQ+cOVNOrltXXtZZ+T2mKFTFaWaN1F7MMH+xIX10dcnsWzq5o0Qz/tpwEfFOaQ5j43ez5BJdE5yD" + "wj4sDIdY57jAQ4141jU2oHpJCjY5Obhd4692k/FRtN7IMWMn5rQS+uTctWOh/27qatdy+QeAMPZseT4Yoi3W6W//Um9+sJY4" + "RjJ0Q27Fjb/wEqBkGsqx1sIidQc19sZWa2jQxVgG3T8xHYCMbgkqT/OHJaHGa4gxo78ajhdQnGYNWtvuRlGDpdUsxcNfF+6z" + "kbqWgcxKleRnQ1jKKZKS9GeiLoJnBRk8wIgmnrleblMB0GRj8i124HKLTfAm3WHlqy3xpACI3AIxPbk1GrATdkrb496eecVG" + "ko9YTrsdgOn5RV6Ggy/Rxr5wAAvPKBrqx/u1Whyr05pge9AZcGAtFRIZMWRCBnNg+oHE8M1EVipx3wQQXFDFO27Jpr3IandO" + "GgpbjbpwcW0fnDKLw7Ndt8SCyyVUEZwZ4Vp4tXJCOv9rzIMHJi+68Upwp5ILirrBFT5vKs//ub6l5hHzEfZty66+DvXyrpqT" + "VLYRnYMEO10Te24QyYVCO4qSnbJzfwTv5Kp4ODP2ONdSuJgkyYy78MzZ926H1/8rOlTEbWpAYb+QJwrt1nBJBkWRNDi2QNrl" + "UlFKTXZNLNXTP/rHHqfw1us8bhwe4Up2BbECPO+8F0tgSifXusnShtsRwpW7o+z/iXpnwuWePH92YiORo43mdoG4MJ54Emwf" + "B7x9ftguHpq1ZDOBoa3frE8l8a5RIEHemkzQsLlPyZSB+eQgnXrToXwUeEfR3EHG0vSCfokMLk0HinhWGa6hgIyJZQAdUIg4" + "GALvjSsrGEHwJaeLNNn/9EscJOEGyFQFMEy+sVIkL6Pj2+iZuG0OKahk" +) + +GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "I need to inspect `shop/validation.py` next to examine how `validate_quantity` handles zero, negative, and non-integer quantities.", + }, + { + "functionCall": { + "name": "read_file", + "args": { + "end_line": 40, + "path": "shop/validation.py", + "start_line": 1, + }, + "id": "call_412671", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 64, + "totalTokenCount": 739, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 321, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "1C3RatW-Jtuc_uMPhaLroA0", +} + +GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_THOUGHT_SIGNATURE: Final = ( + "Eo0LCooLAWkUfRMuNip0QePQVa4NCyMrc0dwJR2qI/PSRcyScXfwpZhjBJVTorCs5ei6D4xzVf9attfTHx8bOBCX9wXynQ6V" + "JxU6uN562z5VAyrgoGv2nVGJk/mT7SnzIFiOc4RN+xebNGO0BIDf95xyOMeYP+ZHXquqtx1SnQXw7V6p81wTA4cGMnqjNE7F" + "Vs7OqlcpvzeFUmE8gl/Cy1a+u5SgLHwfnvOR3uzIX3u8kdk//dzvCKqXYaRStDf0dsrJICex/9WoOBfZKCujLBMK7CCjKEWF" + "nqERtFxXayTUKz1dvvBkcZ+MbhtrG3RFkBegqokQn9lus3icwBAfkZ9b3ps42j/mYw7DfjR+3Dg/Pyz8mgJVRC312QRxt+70" + "yOCU8POm/GTHJAZvvxdm9U4eiYBs3789F5oV34YzZl1+pcE4WVRwiFkB0vi17xy5XJFYTtzuk9d9ySr5Fz1de6nTKWk6TpuX" + "PM4q8bhX51kJWPCr1W4oVn1GhKpCtRyrXzKhuMsWcHXu/yKHUVjxyEFY+cTsVtHfrMHsIhMHyNnBqz2JclGXtFZScaq6AZ8g" + "0JZPqqAXgfgNyx2RaxoaCtITznOlBwMC1hTAZCrtjhHQNvGsAEdaRBAnZRFgeDHkuL7r7D9tr38UdxaUf/R5vfl5eO0ksj4X" + "bdeUG/LAnWP2CLUVjIoF50o4ZHKRy5UT/Mhr45986ePmtWnIN9t6dXOCB5I4qoEaljwkDn+llwZMbPq1orHdSrUAR6flcl3h" + "qVutnVOFgN3NeHvNkrkjUCvckiQL/ZBiG5YeDKfMP+XECDWIoTS2/DCJ7SX6d44B2yJ/PO2PH3nm7hr81U7r7xYKly4/2YFX" + "8bAsFJt4lGihfDHXFOr/VbQQd/CobZtLV9UZP7m1wUGXfL0knP31bdFWDKmSp/2giddbpIsGQf6j7h+YiE3M6QEfvRi7kaDX" + "CQ16O/4XJzG85/BxPw8ssR1cN+xLM95pZkDQO2MN1rxLVogF+T+Xc0M2SSjwbahM4nwYmisOqXv4Wuy0V7hQYJUySCvHabdH" + "nmTi+3mIfs9/zzHT3QUGZvAumBCZYfz1kqyz6mytXSwQbhhktmDLgIaj1abGm2gSn10fxl8pDm/mTq8/owS9aNPyDYChNBMr" + "NGYEBMXA4DtZy9TjybzMXKGPi4d/M7EZM5sC33WnCl2JovveyeZUNPWkZTV0+TjSSqk8C488huCOzv+PgJMvtBq6XiQW9ZV1" + "HQWk8XRMK7VMMhuv9sRqy1Lh/h8kIlSWtaCCNwpqIgUlZ+xybbY9dRGdsWq5FoxCU/OLmlSv7BcqXAamxxfIK9inXRKd/xkI" + "bR7OCeIaFq97Z5M08LfUpu9WjcXGgvtsppOUB4Rik2uCBVSzA/OqQdY1/N3q+BPYCvhOVVEBOJEEAvH9iAQOtCw4peaCtSLL" + "7vLh7s/lIZGBYj5I4zjoOfoSuT9g1x9uvv9h2sJUeP1HsN9Mp95weKBbF3HUN5F/lVRTan2I+Wb2+WnMMbashS1hoVIfk/Bc" + "426M3xPy7JDsczXLUP0PAyhAc2+xiFnkI120l4IHT3JK53AArx7svpTxzLOvPffVIcodkyWXHeJeC6eQbD6Elt/wWbx5Ig1L" + "mJQP7egSoyqmzu0zpuIC0xVsmDdgnumSi7acTjp1mIsLHHLLaWPcpxaNeiwBQf/7FfKsGqJ6p+I+lzt/rdRDOR4apkwKRwnr" + "d5beSf38TJd7L8Zj1qXp36Q2rnvtKubSm7f451w9ZbKsqbMDrwoagNWsfG9kc0CR0wtraSRzBHD6GUfoapPbplOLsSyjzEvX" + "i48aG814wRmQk932cd8ETx1a9BIbN/bsOr6stttugYakpvD3b8ckhfXW7eXfpKywBBzu1s1KLrU=" +) + +GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_FRAMES: Final = ( + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "I need to read `shop/validation.py` next to see how `validate_quantity` currently handles zero, negative, and", + }, + ], + "role": "model", + }, + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 27, + "totalTokenCount": 676, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 295, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "2C3RavD_A77W-sAP9brxyQk", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "text": " non-integer quantities.", + }, + ], + "role": "model", + }, + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 32, + "totalTokenCount": 681, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 295, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "2C3RavD_A77W-sAP9brxyQk", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "read_file", + "args": { + "start_line": 1, + "end_line": 50, + "path": "shop/validation.py", + }, + "id": "call_654547", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 65, + "totalTokenCount": 714, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 295, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "2C3RavD_A77W-sAP9brxyQk", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "", + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 65, + "totalTokenCount": 714, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 295, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "2C3RavD_A77W-sAP9brxyQk", + } + ), +) + +ADAPTIVE_THINKING: Final[Mapping[str, JsonValue]] = {"type": "adaptive"} + +GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE: Final = replace( + GEMINI_3_8_FLASH_TEST_CASE, + scenario="thoughts_off", + litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "thinking": dict(ADAPTIVE_THINKING)}, + expected_provider_request=with_thinking_config( + GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, {"thinkingLevel": "high", "includeThoughts": False} + ), + mock_provider_response=GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE, + expected_litellm_response={ + **GEMINI_MESSAGES_RESPONSE, + "id": "1C3RatW-Jtuc_uMPhaLroA0", + "model": "gemini/gemini-3.8-flash", + "content": [ + { + "type": "text", + "text": ( + "I need to inspect `shop/validation.py` next to examine how `validate_quantity` handles zero, " + "negative, and non-integer quantities." + ), + }, + { + "type": "tool_use", + "id": "call_412671", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "provider_specific_fields": {"signature": GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE}, + }, + ], + "usage": {"input_tokens": 354, "output_tokens": 385}, + }, +) + + +def with_thinking(case: TranslationTestCase, scenario: str, field: str, value: JsonValue) -> TranslationTestCase: + return replace( + case, + scenario=scenario, + litellm_request={**case.litellm_request, "thinking": {**ADAPTIVE_THINKING, field: value}}, + ) + + +THINKING_ASKS: Final = ( + with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_summarized", "display", "summarized"), + with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_updates", "display", "updates"), + with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_int", "display", 7), + with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_empty", "display", ""), + with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_list", "display", ["omitted"]), + with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_long", "display", LONG_SUMMARY), + with_thinking(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "display_omitted", "display", "omitted"), + with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "summary_auto", "summary", "auto"), + with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "summary_empty", "summary", ""), + with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "summary_long", "summary", LONG_SUMMARY), + with_thinking(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_int", "summary", 7), + with_thinking(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_list", "summary", ["auto"]), + with_thinking(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_null", "summary", None), +) + + +def test_gemini_keeps_thinking_and_billing_reasoning_without_thought_text_when_the_gateway_turns_it_off( + thoughts_off: Gateway, provider: SharedProvider +) -> None: + assert_translation(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, thoughts_off, provider) + + +@pytest.mark.parametrize("case", THINKING_ASKS, ids=tuple(case.id for case in THINKING_ASKS)) +def test_a_thinking_display_or_summary_ask_brings_thought_text_back_over_the_gateway_default( + case: TranslationTestCase, thoughts_off: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, thoughts_off, provider) + + +def test_display_omitted_alone_keeps_thought_text_on_the_default_gateway( + gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation( + with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_omitted_default", "display", "omitted"), gateway, provider + ) + + +def test_a_streamed_turn_keeps_thinking_and_billing_reasoning_without_thought_text( + thoughts_off: Gateway, provider: SharedProvider +) -> None: + provider.expect( + Reply( + content_type="text/event-stream", + chunks=tuple(encode_sse_frame(frame) for frame in GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_FRAMES), + ) + ) + response: Final = thoughts_off.request( + "POST", "/v1/messages", {**GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE.litellm_request, "stream": True} + ) + assert_provider_request( + provider.received(), + "/models/gemini-3.8-flash:streamGenerateContent?alt=sse", + GEMINI_3_8_FLASH_TEST_CASE.expected_provider_headers, + GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE.expected_provider_request, + ) + assert response.status_code == 200, response.text + events: Final = tuple(object_value(frame.data) for frame in parse_sse_frames(response.text)) + started: Final = tuple( + object_value(event["content_block"])["type"] for event in events if event.get("type") == "content_block_start" + ) + assert started == ("text", "tool_use") + deltas: Final = tuple( + object_value(event["delta"]) for event in events if event.get("type") == "content_block_delta" + ) + assert "".join(str(delta.get("text", "")) for delta in deltas) == ( + "I need to read `shop/validation.py` next to see how `validate_quantity` currently handles zero, negative, " + "and non-integer quantities." + ) + assert [object_value(event["usage"]) for event in events if event.get("type") == "message_delta"][-1] == { + "input_tokens": 354, + "output_tokens": 360, + } diff --git a/tests/integration/translation/responses/reasoning/__init__.py b/tests/integration/translation/responses/reasoning/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/responses/reasoning/test_responses_reasoning_gemini.py b/tests/integration/translation/responses/reasoning/test_responses_reasoning_gemini.py new file mode 100644 index 00000000000..984f7d2a8c4 --- /dev/null +++ b/tests/integration/translation/responses/reasoning/test_responses_reasoning_gemini.py @@ -0,0 +1,400 @@ +from collections.abc import Mapping +from dataclasses import replace +from typing import Final +from unittest.mock import ANY + +import pytest +from integration._support.client import Gateway, list_value, object_value +from integration._support.process import graceful_stop_seconds +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from integration.translation.case import SSEFrame, TranslationTestCase +from integration.translation.gemini_thoughts import LONG_SUMMARY, thoughts_off, with_thinking_config +from integration.translation.responses.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE, GEMINI_RESPONSES_RESPONSE +from integration.translation.runner import ( + assert_provider_request, + assert_translation, + encode_sse_frame, + parse_sse_frames, +) +from pydantic import JsonValue + +__all__ = ("thoughts_off",) + +pytestmark = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE: Final = ( + "EvQLCvELAWkUfRM8dwxvoWqyX5HLMaxcqWN9o4LPX6M6MZM/OD962KfMSvacFOxwOEb2B3rQM7QgoyT58YrItR4zpDSZJUxA" + "N4shekmtkSRpwrqE0rmH1v9AE8w5lR0Jewj8xmiiUXUrK0qvpyxhbVJRmIgdt9CqW+B4bJHYpp1VFAuHMk8eyBRoVNXakpsG" + "xjvHWGDomXbyJ7XuG1uHzvA5QtV2lgKUwRNvQqbFgbMhKN4dc8Imm2kwCLQONRN6wYIKJnYIi6ib+MhsFY+hZ6sPysMzVyrk" + "nGhc6pZwOKHbUjAraY/X98HO7UEL/q86LKUIvKTS4FfZ8keuyIQQBMdX68fdZPqurR4fI2dr5pFSDbaJWEVaE9XAY9wOXhOE" + "uC5SxY0U7nPU39eycNqAoAf+O7i/uHn1eCfLMhvAZyoH0z6vMfofnVn6L6alZtnRFQ3rEdJWEZHFmsLOk7G9SHlLAYKQjEdj" + "5aQHsKqBG1SFODPWSeKduI5w/kgFwgrw1ye1rnFTd+FDBs3DsQUlwh+jbPg8tr3KnCL5f2bxoGShUanPcGRAJ4b/IWGM3+wf" + "Whcx+3JY/OfuklkCnw86uRlcJiEjWd9XZ+jAby3ECZxBuXsHSE+Ynzpd0mp5g/S1orzqlmUJ/PlOcbDR/AsFduZvZwG0n4hq" + "hbXIIPszIpEyGDhkC//vxXsWdo5gLcsnEoUlLCESN3Sc12goKneId6u8pVWfzvHwRa0pK0LIWo4GcWZepZpEoYARlUZN6Dat" + "FNJ0nFW1sGaNb4CEYdSSp3m+GVobHpdz8qsGJxWrzzc6ZYmO7LbDJ25dJtyCl9QcfyzGbkFVDdAaTBEM23EdyZMFc2IIT24K" + "OVx5esCx+VjDZOmOzue/ZR0xrfaPXVa0CMuzGA8WwiphpWTbD8xZUnqEUZQNwKrjU91+SIhaBKcTVM8F1Wj0t9vlQkkMlzth" + "bJxIw+Ccu51gh/s8/awIDsBwC2Ax6eYIhUoWxUWYcphX7r3lBKhIuzv7FyO8tmaRkIOy6Eb1lXHB4fHAIOimZ/n4ClgK/KOZ" + "XVLosaGMKsmjYAdrxxPaNqs+say+pQ08bec/c+mqWeDbTOJ2JrCKM/Q+EcyD+82QqWlO68boeJyyt0fIvTCAdn57xN5Th7r1" + "WGhT9YHeSB1Y77P9FaJLQqAOjr+yogG7suiCP4QHg1arBlVzvaxAdnlBeXSv6pbQJuuvsFV9+SfwKHVXaBv68bcjSYKm8YZa" + "vfK57jweJUxvknCUpzrDRBdVZX28H60eQ3X6S0cV/696AO8NHoKuMv9zsCyu4WXWnpYEObtVD4Sn5hdSn1tnOkyUTLmN+bHV" + "sRGe5WTmrgnTKbrGDqpeqFpYLeMftkKWX+DHZj/LvZiNH1pHJid4nbEvcUKHciculasWiLZXDhEDKA1uzvSeG7CHjtHziqRi" + "b0dY7pQ0iEJUrjIwMVPnFZoyrywYZVPYgk+jna9FJypscuCz5XzdV+N2b2yrKryU6NhOQ5u/lzMNEFsyYxuXp6LZ5CVgdH1T" + "k+1rUf3sWPTO+CyDj+GzWvo5ljP7HihqyitafZJeKOSfaPfrJpBq+R8jnwDOVLCS1lnkhsof12PFoQHAE0YzGDtVscrstW4O" + "OaEdtNyNh30fhwwaZ/XGM4O9yJ8l7CgUvrgY/t8/f1hB/ZIc980HGs3G/bmVBYTxeZ18rqanrURtVuNtveyGief6eiEVIFjo" + "HWkAFXM+sn1zsJI6l4N9TPMbkoPxLCw2KaOaI/7mQjsH6VktpGuuqpHw3S5tiw3OGwh0lEwydKT0bk4hUhWqoBHqFpCrmKqA" + "g3z0kEHxQjQ1j8GmHusdPp4odrCWphw44m1VeWYjF4VjQ16Vik3SSrC8g3v9sHhtKM+3RL6z8zzukqgcQkXf7RXWvmFttckj" + "yYWM+vfQm/UvCzQ4dKvFHMqgmwuyEISPO9CD68/518wNpbQD1iB/RnxKwyYoD4u9YQ9TzXCzk2eP0NBB6pRmGg5nBo2SP0PB" + "BRZqlxnFX4XfQiS9pxNL" +) + +GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "To see how zero, negative, and non-integer quantities are currently handled and rejected by the validation logic, the next file I need to inspect is `shop/validation.py`.", + }, + { + "functionCall": { + "name": "read_file", + "args": { + "path": "shop/validation.py", + "start_line": 1, + "end_line": 50, + }, + "id": "call_584229", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 70, + "totalTokenCount": 790, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 366, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "2y3RasgqkpyMxw-RxpeQBg", +} + +GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_THOUGHT_SIGNATURE: Final = ( + "EvYSCvMSAWkUfROtzl1tHuKrIElCi7xsI6n+7cPuqsM/qZSYIpK3gONKCCo0E4moadzMQ3sN09ll3gRjbcBg6rKSck+JTBhd" + "mrLsP/sqfxiYfMx4lWzDeq/Pw/IPhT2g5/2EvSU4FDsUczorTsD55gKvgRT8v05YTFMGF/np8r3VT52Q2yAzwRK+31qn8pYS" + "c25if5coTZJrvmJZDlZ9/64aibghRps9zN07yRVXpUjtLyKMaTDH0loGgiQn0I7Cxfhw9oUyZels7Un7pB7omA4RKW5xp5rx" + "Y+WgSeAEyyxNcPIkRvaxHaY1p2QaR3m17a0L7joJYNB+gNg7R/4Mfwpr8WuSOyLfBaKTuC5IiYWLaRm0ibASYzd59pPUyArt" + "mIx7pPfWxAr8/FT8TVQTSSeHXF88x/xJ2WSTSzQ3+FPw6Wm/LpfeGwQPX/g1+8f+KVcj1qAwT/tUobqOh7DdWl68CnNFReX4" + "SwBqRjTzZ276P7l15fhWmnQkiWXmLVsufJi5VXidEVrAsuEyVe7JmpExi3Y+0pn82D9x7lyv/TrOCVQgKwC0Zoq9NfuQxWa0" + "RvqmAv6toMfI8owTyzVoItnoGdhcYamwr7AyECKTuqnzMuylzOx5ttZnRc12/Ey2GQR3UxhyxMNLk01/UDicLnEXFl8Zn56i" + "rj51atVum7mJc48qekcKXQ5Lc3/t277v1d+eLmuiHebshpQwg+qQPHnnfky0WL98w343XDb29wwK3+veqQSjrffv5rQY9Tvq" + "3PMuN1pjU7Z3rL6fZmt5UrhqQQnTf4dA7X7n/aLkpA8SpY6ICDDr2x1z0yW5kMnHfxxk+N+j5eWwoTJuxxy94rLaYj3CKT57" + "XhMthQAoF1DGNpGxhbfC/tVuFE9FkxEBiYXqDz5fy2AU+S2zatT7/9mkFaHLY6hkOCS48X9WBVobxsQZZUnkbxplCzg4Huin" + "gPnl1bUDEprLHfTYgZHcnn3FgWFPwzxNBebrc4xjqXciMgReu4kOLP18JZbIJSjhP/RZAnQJEEOI8OrVGZlmiC7/JJx+8PTX" + "XYMTKtiVU/R3gOhOeVNN9WXruUoQzuy+JfJGg8ZQ2ni0Ugkx0sLO5rDsTOZ67bA+B99FQnQcBLyb40EPTOzrLrcM9wzXL+YZ" + "JUwNtdaQGxh/aUeKex/8of7GUH9ZsCZW0YN5xuEs8e2Q+vi2Msluckvki/wcyx7oOxmjw8bEbUkPBFq/ZEOcQqtetPLBzNPt" + "smJ4EfOU/kwBBLxplt6JrF+0ILeWJuH7OGpvpseXEosLpJ9HV4oIHL9co8EWGH/oAUzcuvhymR8FRziVE2IgVkreryIfHAs/" + "umsJgk1dXZk4A/R7KtjHKau0yxLAYJh6jWLvLsLv5N44VfKCQtyWFdUOSGbqn6E4YuOYVc+x+TGQcSMniobTekwm+UtqzVH8" + "mmtd8nwzwVkDgqZEAHrrnJOALLPeIbPuQ0/zjuUJmQovnnSqUyyCxQDpaxrlLD3iIZp9/9hPTY+lYHlam7VfChpQ7dPQ075t" + "ECiPA5TgMzh1qNQBF+rgKtrcrPo1ZDFqlseAybmSKqnCdvD5vdfdeVL8cmQIQao9qHDVh/aR0uwOsVKKzavPIYH2mhomg/cz" + "XB1nJG2zXB99g08Bd14c/zHHPNeqkqmwURCTxzPh8aeoRPcC8uL/eyawL66FfyUo40og9uridBDA4qx4ZMI+P1dBKFnMPM7o" + "Blpo+Sg9m6XZNPNVbDqRKKmX9OWCng7/PAKX+wa8EzRetNbyh73eMviMZaiN9YAyoKiS+nR5FqrPgNcFMqa8dHDEyY6O3Vuh" + "5Er5R3iGNACXMQvLRDVy/1UmFY1mzwew9lrDcUkZ0MNKBrbtDC87XEFGMejqvoJgF5Yl+q1mJE6gMBDTo6UEP1SOctxHyX23" + "mvqhCK9wOvK91UGnNyd+8ZQtQPP/L99d2y7hHmE6VEA/nUa2qkkO99vEQmJV3ilax+wZknABwk5sFdh7vWLwhEvxkdGYUh6c" + "HnA70VX5Kh0twJPKwE7gNASf/rvE5kYdrei5RtZNKMpVulXMPVho3eRBW6Qf6Ka0JjDZF5Dr8IXXsjP2LFTmnLNcnxu0FXx8" + "4xRuw1+yN3cP5R8LFlSGJmqPf5kFwAwpzT5FAXETiBaj7lg3WyG+OaDpab2NG+BvHbdfrDAuFKSsTDZ3Vt58fj6fuhS8tFd4" + "YzWL6X30pj+r7QRAefk+2CQDKM1TnbAJbyBcG2DVfeWpgmNjtzoV+RXNykIFIy6rSeOnUBAJCXwbAe7Scp5w/lx03a5aooRU" + "CP7x7CAmAU+YuGerG7A/3bbWDqN4EHGawkloK9UXmlOVX7uzLy0q3fJZb/6pSvPX+DR2rjFq0CSKnNFyBXYQRHx+DekAHWJS" + "VGUw7RLAPd4DE2HRiYTss7vtXalkP0hQ/vnt8lS9bjyBCi8ly+4FjFpo/SeKx0TKFulbFp1GPpsNERT+7kEkTsNZQHXWXPRB" + "PH0qmpFWZjI98fzWIGXGF376X0bcg6g67C/K/+YtH+QIU82ScAM0ozzQ1uzb0ci6wnM4JjNKA3ctVX6zuPR20Tb7VLlq5rwR" + "coOKjGQmBGbUiW5aKAAsMnIDhxf2lPCcwwb51C0Zb6hfOG+ka7LLcstgq4fmXHQ11BbTFu9ctAhy0E5V3aYSJAilb6VXZdD4" + "kSl3I0D2Xc/pTLPAzsxg6mEHL7J/DD1lRvEtSh4U34RCwxMk+pQbfEo6y/lkIVhXuE8p7d4mhPtsDJkls/3vnl+rR5ycebjH" + "g/zC01CjHhbVglb+HVEC5FMTrBNy/bweTE6c7qNHV6EWRN64nqvs4IW/jEVoaI2qt9WAJZ1br21rCE01iQagQUs1/ZH2KpQB" + "jTe3ju/BotwiE4TdqHcDL/VP62yq9KLeiBNXY2ctb3NwJzzfLrM2KHv2b4YmWuZ/VLABBfh9r3OzSmnFBvmWsZjZOHHEQXMI" + "hI+WDpRGB54CWmnWs6YzQK70IKrZUtq4BfPQJGf8OqO/ZeCJA4kinQbCaTTqQOJLsONKbPTqm8ImXz/QiML+KFvY3Z3ReW3T" + "NsJ3CGW7NRY1cfBIkhHuKHxPK1Ua5H3b7yP+h8OKrbqyjYPw2kE56TOH75HnIp/Ib6w+7uZpMVIbM4fYCRWlvQ7cUtaRq4fk" + "qqgHykE8bJYoU9eJFmPJNj4gt051sXJoslr2C6mvHYdSZgKxNa1ksd95cPKS3yURYQ==" +) + +GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_FRAMES: Final = ( + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "To understand how zero, negative, and non-integer quantities are checked during validation, I need to inspect", + }, + ], + "role": "model", + }, + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 21, + "totalTokenCount": 847, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 472, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "3S3Rat6pNonj_uMPv9i78Qw", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "text": " `shop/validation.py` next.", + }, + ], + "role": "model", + }, + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 30, + "totalTokenCount": 856, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 472, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "3S3Rat6pNonj_uMPv9i78Qw", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "read_file", + "args": { + "end_line": 50, + "path": "shop/validation.py", + "start_line": 1, + }, + "id": "call_5018", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 63, + "totalTokenCount": 889, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 472, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "3S3Rat6pNonj_uMPv9i78Qw", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "", + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 63, + "totalTokenCount": 889, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 472, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "3S3Rat6pNonj_uMPv9i78Qw", + } + ), +) + +GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE: Final = replace( + GEMINI_3_8_FLASH_TEST_CASE, + scenario="thoughts_off", + expected_provider_request=with_thinking_config( + GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, {"thinkingLevel": "high", "includeThoughts": False} + ), + mock_provider_response=GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE, + expected_litellm_response={ + **GEMINI_RESPONSES_RESPONSE, + "model": "gemini/gemini-3.8-flash", + "output": [ + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": ( + "To see how zero, negative, and non-integer quantities are currently handled and rejected " + "by the validation logic, the next file I need to inspect is `shop/validation.py`." + ), + "annotations": [], + }, + ], + "phase": None, + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}', + "call_id": "call_584229__thought__" + GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE, + "name": "read_file", + "type": "function_call", + "id": "fc_call_584229__thought__" + GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE, + "namespace": None, + "status": "completed", + "provider_specific_fields": {"thought_signature": GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE}, + }, + ], + "usage": { + "input_tokens": 354, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 354, + "video_tokens": None, + }, + "output_tokens": 436, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 366, "text_tokens": 70}, + "total_tokens": 790, + "cost": None, + }, + }, +) + + +def with_summary(case: TranslationTestCase, scenario: str, summary: JsonValue) -> TranslationTestCase: + reasoning: Final[Mapping[str, JsonValue]] = {"effort": "high", "summary": summary} + return replace( + case, + scenario=scenario, + litellm_request={**case.litellm_request, "reasoning": dict(reasoning)}, + expected_litellm_response={**case.expected_litellm_response, "reasoning": dict(reasoning)}, + ) + + +SUMMARY_ASKS: Final = ( + with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_auto", "auto"), + with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_concise", "concise"), + with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_empty", ""), + with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_long", LONG_SUMMARY), + with_summary(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_int", 7), + with_summary(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_list", ["auto"]), + with_summary(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_null", None), +) + + +def test_gemini_keeps_thinking_and_billing_reasoning_without_thought_text_when_the_gateway_turns_it_off( + thoughts_off: Gateway, provider: SharedProvider +) -> None: + assert_translation(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, thoughts_off, provider) + + +@pytest.mark.parametrize("case", SUMMARY_ASKS, ids=tuple(case.id for case in SUMMARY_ASKS)) +def test_a_reasoning_summary_ask_brings_thought_text_back_over_the_gateway_default( + case: TranslationTestCase, thoughts_off: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, thoughts_off, provider) + + +def test_a_streamed_turn_keeps_thinking_and_billing_reasoning_without_thought_text( + thoughts_off: Gateway, provider: SharedProvider +) -> None: + provider.expect( + Reply( + content_type="text/event-stream", + chunks=tuple(encode_sse_frame(frame) for frame in GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_FRAMES), + ) + ) + response: Final = thoughts_off.request( + "POST", "/v1/responses", {**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "stream": True} + ) + assert_provider_request( + provider.received(), + "/models/gemini-3.8-flash:streamGenerateContent?alt=sse", + GEMINI_3_8_FLASH_TEST_CASE.expected_provider_headers, + GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE.expected_provider_request, + ) + assert response.status_code == 200, response.text + events: Final = tuple( + object_value(frame.data) for frame in parse_sse_frames(response.text) if frame.data != "[DONE]" + ) + assert [event["type"] for event in events if str(event["type"]).startswith("response.reasoning")] == [] + assert "".join(str(event["delta"]) for event in events if event["type"] == "response.output_text.delta") == ( + "To understand how zero, negative, and non-integer quantities are checked during validation, I need to " + "inspect `shop/validation.py` next." + ) + completed: Final = object_value( + next(event["response"] for event in events if event["type"] == "response.completed") + ) + assert [object_value(item)["type"] for item in list_value(completed["output"])] == ["message", "function_call"] + assert object_value(completed["usage"])["output_tokens_details"] == { + "reasoning_tokens": 472, + "text_tokens": 63, + } diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index b84529de0d6..91083921130 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -291,6 +291,37 @@ def test_exact_cache_key_includes_anthropic_messages_params(anthropic_param): assert baseline != cache.get_cache_key(model="claude-sonnet-4-5", messages=messages, **anthropic_param) +@pytest.mark.parametrize( + "summary_ask", + [ + {"reasoning_summary": "auto"}, + {"reasoningSummary": "auto"}, + {"extra_body": {"reasoning_summary": "auto"}}, + {"extra_body": {"reasoningSummary": "auto"}}, + ], + ids=["top_level", "top_level_camel", "extra_body", "extra_body_camel"], +) +def test_exact_cache_key_includes_the_reasoning_summary_alias(summary_ask: dict[str, object]) -> None: + """The AI SDK summary alias decides whether thought summaries come back, wherever the request + carries it, so a reply cached without it must not answer a request that asks for one.""" + cache: Final = Cache(type=LiteLLMCacheType.LOCAL) + messages: Final = [{"role": "user", "content": "how much is the ball?"}] + no_ask: Final = cache.get_cache_key(model="gemini/gemini-3.8-flash", messages=messages, reasoning_effort="medium") + ask: Final = cache.get_cache_key( + model="gemini/gemini-3.8-flash", messages=messages, reasoning_effort="medium", **summary_ask + ) + assert no_ask != ask + assert ask == cache.get_cache_key( + model="gemini/gemini-3.8-flash", messages=messages, reasoning_effort="medium", **summary_ask + ) + assert ask != cache.get_cache_key( + model="gemini/gemini-3.8-flash", + messages=messages, + reasoning_effort="medium", + reasoning_summary="detailed", + ) + + @pytest.mark.asyncio async def test_embedding_cache_skips_write_when_one_input_yields_many_embeddings(monkeypatch): """A cross-encoder behind /embeddings returns one score per document for a single @@ -561,10 +592,7 @@ def test_caching_v2(): # test in memory cache litellm.cache = None # disable cache litellm.success_callback = [] litellm._async_success_callback = [] - if ( - response2["choices"][0]["message"]["content"] - != response1["choices"][0]["message"]["content"] - ): + if response2["choices"][0]["message"]["content"] != response1["choices"][0]["message"]["content"]: print(f"response1: {response1}") print(f"response2: {response2}") pytest.fail(f"Error occurred:") @@ -595,10 +623,7 @@ def test_caching_with_ttl(): litellm.cache = None # disable cache litellm.success_callback = [] litellm._async_success_callback = [] - assert ( - response2["choices"][0]["message"]["content"] - != response1["choices"][0]["message"]["content"] - ) + assert response2["choices"][0]["message"]["content"] != response1["choices"][0]["message"]["content"] except Exception as e: print(f"error occurred: {traceback.format_exc()}") pytest.fail(f"Error occurred: {e}") @@ -632,9 +657,7 @@ def test_caching_with_default_ttl(): def test_caching_with_models_v2(): - messages = [ - {"role": "user", "content": "who is ishaan CTO of litellm from litellm 2023"} - ] + messages = [{"role": "user", "content": "who is ishaan CTO of litellm from litellm 2023"}] litellm.cache = Cache() print("test2 for caching") litellm.set_verbose = True @@ -657,18 +680,12 @@ def test_caching_with_models_v2(): litellm.cache = None litellm.success_callback = [] litellm._async_success_callback = [] - if ( - response3["choices"][0]["message"]["content"] - == response2["choices"][0]["message"]["content"] - ): + if response3["choices"][0]["message"]["content"] == response2["choices"][0]["message"]["content"]: # if models are different, it should not return cached response print(f"response2: {response2}") print(f"response3: {response3}") pytest.fail(f"Error occurred:") - if ( - response1["choices"][0]["message"]["content"] - != response2["choices"][0]["message"]["content"] - ): + if response1["choices"][0]["message"]["content"] != response2["choices"][0]["message"]["content"]: print(f"response1: {response1}") print(f"response2: {response2}") pytest.fail(f"Error occurred:") @@ -708,9 +725,7 @@ def test_get_cache_key(): cache_key = cache_instance.get_cache_key( **{ "model": "gpt-3.5-turbo", - "messages": [ - {"role": "user", "content": "write a one sentence poem about: 7510"} - ], + "messages": [{"role": "user", "content": "write a one sentence poem about: 7510"}], "max_tokens": 40, "temperature": 0.2, "stream": True, @@ -721,9 +736,7 @@ def test_get_cache_key(): cache_key_2 = cache_instance.get_cache_key( **{ "model": "gpt-3.5-turbo", - "messages": [ - {"role": "user", "content": "write a one sentence poem about: 7510"} - ], + "messages": [{"role": "user", "content": "write a one sentence poem about: 7510"}], "max_tokens": 40, "temperature": 0.2, "stream": True, @@ -736,9 +749,9 @@ def test_get_cache_key(): # Hexadecimal representation of the hash hash_hex = hash_object.hexdigest() assert cache_key == hash_hex - assert ( - cache_key_2 == hash_hex - ), f"{cache_key} != {cache_key_2}. The same kwargs should have the same cache key across runs" + assert cache_key_2 == hash_hex, ( + f"{cache_key} != {cache_key_2}. The same kwargs should have the same cache key across runs" + ) embedding_cache_key = cache_instance.get_cache_key( **{ @@ -756,15 +769,13 @@ def test_get_cache_key(): print(embedding_cache_key) - embedding_cache_key_str = ( - "model: azure/text-embedding-ada-002input: ['hi who is ishaan']" - ) + embedding_cache_key_str = "model: azure/text-embedding-ada-002input: ['hi who is ishaan']" hash_object = hashlib.sha256(embedding_cache_key_str.encode()) # Hexadecimal representation of the hash hash_hex = hash_object.hexdigest() - assert ( - embedding_cache_key == hash_hex - ), f"{embedding_cache_key} != 'model: azure/text-embedding-ada-002input: ['hi who is ishaan']'. The same kwargs should have the same cache key across runs" + assert embedding_cache_key == hash_hex, ( + f"{embedding_cache_key} != 'model: azure/text-embedding-ada-002input: ['hi who is ishaan']'. The same kwargs should have the same cache key across runs" + ) # Proxy - embedding cache, test if embedding key, gets model_group and not model embedding_cache_key_2 = cache_instance.get_cache_key( @@ -817,9 +828,7 @@ def test_get_cache_key(): ) print(embedding_cache_key_2) - embedding_cache_key_str_2 = ( - "model: EMBEDDING_MODEL_GROUPinput: ['hi who is ishaan']" - ) + embedding_cache_key_str_2 = "model: EMBEDDING_MODEL_GROUPinput: ['hi who is ishaan']" hash_object = hashlib.sha256(embedding_cache_key_str_2.encode()) # Hexadecimal representation of the hash hash_hex = hash_object.hexdigest() @@ -852,9 +861,7 @@ def test_redis_caching_multiple_namespaces(): # Mock the Redis client creation from the _redis module with ( patch("litellm._redis.get_redis_client") as mock_get_redis_client, - patch( - "litellm._redis.get_redis_connection_pool" - ) as mock_get_redis_connection_pool, + patch("litellm._redis.get_redis_connection_pool") as mock_get_redis_connection_pool, ): # Create a mock Redis client that simulates real Redis behavior mock_redis_client = MagicMock() @@ -927,18 +934,10 @@ def test_redis_caching_multiple_namespaces(): mock_response="Response without namespace", ) - print( - f"Response 1 type: {type(response_1)} - ID: {getattr(response_1, 'id', 'N/A')}" - ) - print( - f"Response 2 type: {type(response_2)} - ID: {getattr(response_2, 'id', 'N/A')}" - ) - print( - f"Response 3 type: {type(response_3)} - Cache hit: {isinstance(response_3, str)}" - ) - print( - f"Response 4 type: {type(response_4)} - ID: {getattr(response_4, 'id', 'N/A')}" - ) + print(f"Response 1 type: {type(response_1)} - ID: {getattr(response_1, 'id', 'N/A')}") + print(f"Response 2 type: {type(response_2)} - ID: {getattr(response_2, 'id', 'N/A')}") + print(f"Response 3 type: {type(response_3)} - Cache hit: {isinstance(response_3, str)}") + print(f"Response 4 type: {type(response_4)} - ID: {getattr(response_4, 'id', 'N/A')}") print(f"Redis storage keys: {list(redis_storage.keys())}") @@ -947,10 +946,7 @@ def test_redis_caching_multiple_namespaces(): namespace_1_keys = [k for k in cache_keys if k.startswith(f"{namespace_1}:")] namespace_2_keys = [k for k in cache_keys if k.startswith(f"{namespace_2}:")] no_namespace_keys = [ - k - for k in cache_keys - if not k.startswith(f"{namespace_1}:") - and not k.startswith(f"{namespace_2}:") + k for k in cache_keys if not k.startswith(f"{namespace_1}:") and not k.startswith(f"{namespace_2}:") ] print(f"Namespace 1 keys: {namespace_1_keys}") @@ -963,9 +959,7 @@ def test_redis_caching_multiple_namespaces(): assert len(no_namespace_keys) > 0, "Should have cache keys for no namespace" # The main test: response 3 should be a cache hit (string) because it uses same namespace as response 1 - assert isinstance( - response_3, str - ), "Response 3 should be a cache hit (string) for same namespace" + assert isinstance(response_3, str), "Response 3 should be a cache hit (string) for same namespace" # response 1 & 2 should be ModelResponse objects (cache misses) assert hasattr(response_1, "id"), "Response 1 should be a ModelResponse object" @@ -973,14 +967,14 @@ def test_redis_caching_multiple_namespaces(): assert hasattr(response_4, "id"), "Response 4 should be a ModelResponse object" # response 1 & 2 should have different IDs (different namespaces) - assert ( - response_1.id != response_2.id - ), f"Expected different response ID for different namespace. Got {response_1.id} and {response_2.id}" + assert response_1.id != response_2.id, ( + f"Expected different response ID for different namespace. Got {response_1.id} and {response_2.id}" + ) # response 1 & 4 should have different IDs (different namespaces) - assert ( - response_1.id != response_4.id - ), f"Expected different response ID for no namespace vs namespaced. Got {response_1.id} and {response_4.id}" + assert response_1.id != response_4.id, ( + f"Expected different response ID for no namespace vs namespaced. Got {response_1.id} and {response_4.id}" + ) _TOOL_TURN_ITEM: Final = {"role": "user", "content": "hi"} @@ -1305,12 +1299,8 @@ async def test_embedding_caching_individual_items(monkeypatch: pytest.MonkeyPatc monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) item: Final = f"individual embedding {uuid.uuid4().hex}" - first: Final = await aembedding( - model="text-embedding-3-small", input=item, caching=True, mock_response="0.1,0.2" - ) - second: Final = await aembedding( - model="text-embedding-3-small", input=item, caching=True, mock_response="0.3,0.4" - ) + first: Final = await aembedding(model="text-embedding-3-small", input=item, caching=True, mock_response="0.1,0.2") + second: Final = await aembedding(model="text-embedding-3-small", input=item, caching=True, mock_response="0.3,0.4") assert second.data[0].embedding == first.data[0].embedding assert second._hidden_params["cache_hit"] is True @@ -1448,9 +1438,7 @@ async def test_caching_with_cache_controls(monkeypatch: pytest.MonkeyPatch) -> N @pytest.mark.asyncio -async def test_embedding_caching_redis_ttl( - monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None -) -> None: +async def test_embedding_caching_redis_ttl(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None: mock_pipeline: Final = AsyncMock() mock_set: Final = AsyncMock() mock_pipeline.__aenter__.return_value.set = mock_set @@ -1506,18 +1494,14 @@ async def test_logging_turn_off_message_logging_streaming( assert cached_response["choices"][0]["message"]["content"] == "cached" -def test_redis_caching_default_ttl( - monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None -) -> None: +def test_redis_caching_default_ttl(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None: monkeypatch.setattr(litellm, "default_redis_ttl", 120) cache: Final = RedisCache(host="127.0.0.1", port="6379") assert cache.default_ttl == 120 -def test_redis_caching_llm_caching_ttl( - monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None -) -> None: +def test_redis_caching_llm_caching_ttl(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None: monkeypatch.setattr(litellm, "default_redis_ttl", 120) cache: Final = RedisCache(host="127.0.0.1", port="6379") set_cache: Final = MagicMock() @@ -1529,9 +1513,7 @@ def test_redis_caching_llm_caching_ttl( @pytest.mark.asyncio -async def test_redis_caching_ttl_pipeline( - monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None -) -> None: +async def test_redis_caching_ttl_pipeline(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None: monkeypatch.setattr(litellm, "default_redis_ttl", 120) cache: Final = RedisCache(host="127.0.0.1", port="6379") pipeline: Final = AsyncMock() @@ -1551,9 +1533,7 @@ async def test_redis_caching_ttl_pipeline( @pytest.mark.asyncio -async def test_redis_caching_ttl_sadd( - monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None -) -> None: +async def test_redis_caching_ttl_sadd(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None: monkeypatch.setattr(litellm, "default_redis_ttl", 120) cache: Final = RedisCache(host="127.0.0.1", port="6379") redis_client: Final = AsyncMock() diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py index 0dfe4a93649..4b96d2e7991 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py @@ -6,13 +6,17 @@ regression they guard is the one a caller sees: a tier the proxy advertises has leaves the adapter, in the shape the target expects. """ +from collections.abc import Mapping from typing import Final import pytest +import litellm from litellm.llms.anthropic.pass_through.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig +from litellm.utils import reasoning_effort_with_summary_alias MESSAGES = [{"role": "user", "content": "hello"}] @@ -38,14 +42,14 @@ def _reasoning_effort_sent(model: str, provider: str, reasoning_effort: object) return completion_kwargs.get("reasoning_effort") -def _reasoning_effort_sent_for_thinking( +def _completion_kwargs_for_thinking( model: str, provider: str | None, thinking: dict[str, object], *, tools: list[dict[str, object]] | None = None, api_base: str | None = None, -) -> object: +) -> Mapping[str, object]: extra_kwargs: Final = { key: value for key, value in (("custom_llm_provider", provider), ("api_base", api_base)) if value is not None } @@ -66,7 +70,20 @@ def _reasoning_effort_sent_for_thinking( output_format=None, extra_kwargs=extra_kwargs, ) - return completion_kwargs.get("reasoning_effort") + return completion_kwargs + + +def _reasoning_effort_sent_for_thinking( + model: str, + provider: str | None, + thinking: dict[str, object], + *, + tools: list[dict[str, object]] | None = None, + api_base: str | None = None, +) -> object: + return _completion_kwargs_for_thinking(model, provider, thinking, tools=tools, api_base=api_base).get( + "reasoning_effort" + ) SUMMARIZED_THINKING = {"type": "enabled", "budget_tokens": 4096, "summary": "auto"} @@ -78,6 +95,66 @@ MULTIPLY_TOOL = { } +class TestAThinkingSummaryAskReachesGemini: + """Gemini maps a summary ask to ``includeThoughts``, so ``gemini_include_thoughts_default`` only + decides for a ``/v1/messages`` request that asks for nothing, the same as on chat completions.""" + + @pytest.mark.parametrize( + "model, provider", + [("gemini/gemini-3.1-pro-preview", "gemini"), ("vertex_ai/gemini-2.5-flash", "vertex_ai")], + ) + @pytest.mark.parametrize( + "thinking, sends_thoughts", + [ + (SUMMARIZED_THINKING, True), + ({**PLAIN_THINKING, "display": "summarized"}, True), + ({**PLAIN_THINKING, "display": "updates"}, True), + ({**PLAIN_THINKING, "display": "omitted"}, False), + (PLAIN_THINKING, False), + ({"type": "disabled", "display": "summarized"}, False), + ], + ) + def test_the_request_ask_beats_the_gateway_default( + self, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + model: str, + provider: str, + thinking: dict[str, object], + sends_thoughts: bool, + ) -> None: + monkeypatch.setattr(litellm, "gemini_include_thoughts_default", False) + + sent: Final = _completion_kwargs_for_thinking(model, provider, thinking) + gemini: Final = VertexGeminiConfig() + reasoning_effort: Final = reasoning_effort_with_summary_alias( + sent.get("reasoning_effort"), sent, accepts_summary=gemini.accepts_reasoning_summary_in_reasoning_effort + ) + mapped: Final = gemini.map_openai_params( + {"reasoning_effort": reasoning_effort}, {}, model.split("/", 1)[1], True + ) + + assert mapped["thinkingConfig"]["includeThoughts"] is sends_thoughts + assert not isinstance(sent.get("reasoning_effort"), dict) + + def test_auto_summary_leaves_the_gateway_default_in_charge( + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "gemini_include_thoughts_default", False) + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") + + sent: Final = _reasoning_effort_sent_for_thinking("gemini/gemini-3.1-pro-preview", "gemini", PLAIN_THINKING) + + assert sent == "high" + + def test_a_display_ask_leaves_other_chat_targets_on_the_plain_tier(self, local_model_cost_map: None) -> None: + sent: Final = _reasoning_effort_sent_for_thinking( + "databricks/databricks-qwen35-122b-a10b", "databricks", {**PLAIN_THINKING, "display": "summarized"} + ) + + assert sent == "high" + + class TestTheSummaryWrappingOnlyRidesTheResponsesBridge: """Only the Responses API takes ``reasoning_effort`` as a dict. Databricks answered the wrapped ``{"effort", "summary"}`` with ``field 'reasoning_effort' expects input with json type 'string' @@ -100,7 +177,9 @@ class TestTheSummaryWrappingOnlyRidesTheResponsesBridge: ) -> None: monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") - sent = _reasoning_effort_sent_for_thinking("databricks/databricks-qwen35-122b-a10b", "databricks", PLAIN_THINKING) + sent: Final = _reasoning_effort_sent_for_thinking( + "databricks/databricks-qwen35-122b-a10b", "databricks", PLAIN_THINKING + ) assert sent == "high" @@ -113,7 +192,7 @@ class TestTheSummaryWrappingOnlyRidesTheResponsesBridge: ], ) def test_a_bridged_target_keeps_the_summary(self, local_model_cost_map: None, model: str, provider: str) -> None: - sent = _reasoning_effort_sent_for_thinking(model, provider, SUMMARIZED_THINKING) + sent: Final = _reasoning_effort_sent_for_thinking(model, provider, SUMMARIZED_THINKING) assert sent == {"effort": "high", "summary": "auto"} @@ -122,7 +201,7 @@ class TestTheSummaryWrappingOnlyRidesTheResponsesBridge: ) -> None: monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") - sent = _reasoning_effort_sent_for_thinking("azure/responses/gpt-5-mini", "azure", PLAIN_THINKING) + sent: Final = _reasoning_effort_sent_for_thinking("azure/responses/gpt-5-mini", "azure", PLAIN_THINKING) assert sent == {"effort": "high", "summary": "detailed"} @@ -139,21 +218,21 @@ class TestTheSummaryWrappingOnlyRidesTheResponsesBridge: """``completion()`` keeps a gpt-5.5 deployment with function tools on Foundry's chat route when its ``api_base`` is a Foundry OpenAI host, and bridges it to Responses when the base makes it an Azure OpenAI deployment. The adapter has to read the same ``api_base`` to land on the same call.""" - sent = _reasoning_effort_sent_for_thinking( + sent: Final = _reasoning_effort_sent_for_thinking( "azure_ai/gpt-5.5", "azure_ai", SUMMARIZED_THINKING, tools=[MULTIPLY_TOOL], api_base=api_base ) assert sent == expected def test_a_provider_resolved_from_the_api_base_gets_the_plain_tier(self, local_model_cost_map: None) -> None: - sent = _reasoning_effort_sent_for_thinking( + sent: Final = _reasoning_effort_sent_for_thinking( "kimi-k3", None, SUMMARIZED_THINKING, api_base="https://api.together.xyz/v1" ) assert sent == "high" def test_a_chained_gateway_keeps_the_dict_for_its_own_bridge(self, local_model_cost_map: None) -> None: - sent = _reasoning_effort_sent_for_thinking("litellm_proxy/gpt-5.4", "litellm_proxy", SUMMARIZED_THINKING) + sent: Final = _reasoning_effort_sent_for_thinking("litellm_proxy/gpt-5.4", "litellm_proxy", SUMMARIZED_THINKING) assert sent == {"effort": "high", "summary": "auto"} @@ -192,7 +271,7 @@ class TestTheNormalizedTierIsTheTierSent: def test_the_dict_form_normalizes_effort_and_keeps_its_siblings( self, local_model_cost_map, model, provider, expected ): - sent = _reasoning_effort_sent(model, provider, {"effort": "max", "summary": "detailed"}) + sent: Final = _reasoning_effort_sent(model, provider, {"effort": "max", "summary": "detailed"}) assert sent == {"effort": expected, "summary": "detailed"} @@ -215,8 +294,8 @@ class TestTheNormalizedTierIsTheTierSent: ], ) def test_an_azure_hosted_astra_deployment_drops_to_the_tier_it_accepts( - self, local_model_cost_map, model, provider - ): + self, local_model_cost_map: None, model: str, provider: str + ) -> None: """The deployment answers ``max`` with a 400 naming ``none`` through ``xhigh``, so the rows say so and the adapter sends the tier below instead of the rejected one.""" assert _reasoning_effort_sent(model, provider, "max") == "xhigh" diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 40ede74b98b..5c0e9f1a305 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1,6 +1,7 @@ import asyncio, importlib, os, uuid import json import re +from collections.abc import Awaitable, Callable, Mapping from copy import deepcopy from typing import Final, List, Literal, cast, get_args from unittest.mock import MagicMock, Mock, patch @@ -25,6 +26,7 @@ from litellm.types.llms.vertex_ai import GeminiFinishReason, UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage from litellm.utils import _invalidate_model_cost_lowercase_map, CustomStreamWrapper from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.integrations.custom_logger import CustomLogger from litellm.llms.vertex_ai.gemini.transformation import ( gemini_convert_messages_with_history, ) @@ -38,62 +40,62 @@ from litellm.llms.vertex_ai.context_caching.transformation import ( import tempfile GEMINI_3_IMAGE_SIZE_MAPPINGS: Final[tuple[tuple[str, str, str], ...]] = ( - ('512x512', '1:1', '512'), - ('1024x1024', '1:1', '1K'), - ('2048x2048', '1:1', '2K'), - ('4096x4096', '1:1', '4K'), - ('256x1024', '1:4', '512'), - ('512x2048', '1:4', '1K'), - ('1024x4096', '1:4', '2K'), - ('2048x8192', '1:4', '4K'), - ('192x1536', '1:8', '512'), - ('384x3072', '1:8', '1K'), - ('768x6144', '1:8', '2K'), - ('1536x12288', '1:8', '4K'), - ('424x632', '2:3', '512'), - ('848x1264', '2:3', '1K'), - ('1696x2528', '2:3', '2K'), - ('3392x5056', '2:3', '4K'), - ('632x424', '3:2', '512'), - ('1264x848', '3:2', '1K'), - ('2528x1696', '3:2', '2K'), - ('5056x3392', '3:2', '4K'), - ('448x600', '3:4', '512'), - ('896x1200', '3:4', '1K'), - ('1792x2400', '3:4', '2K'), - ('3584x4800', '3:4', '4K'), - ('1024x256', '4:1', '512'), - ('2048x512', '4:1', '1K'), - ('4096x1024', '4:1', '2K'), - ('8192x2048', '4:1', '4K'), - ('600x448', '4:3', '512'), - ('1200x896', '4:3', '1K'), - ('2400x1792', '4:3', '2K'), - ('4800x3584', '4:3', '4K'), - ('464x576', '4:5', '512'), - ('928x1152', '4:5', '1K'), - ('1856x2304', '4:5', '2K'), - ('3712x4608', '4:5', '4K'), - ('576x464', '5:4', '512'), - ('1152x928', '5:4', '1K'), - ('2304x1856', '5:4', '2K'), - ('4608x3712', '5:4', '4K'), - ('1536x192', '8:1', '512'), - ('3072x384', '8:1', '1K'), - ('6144x768', '8:1', '2K'), - ('12288x1536', '8:1', '4K'), - ('384x688', '9:16', '512'), - ('768x1376', '9:16', '1K'), - ('1536x2752', '9:16', '2K'), - ('3072x5504', '9:16', '4K'), - ('688x384', '16:9', '512'), - ('1376x768', '16:9', '1K'), - ('2752x1536', '16:9', '2K'), - ('5504x3072', '16:9', '4K'), - ('792x336', '21:9', '512'), - ('1584x672', '21:9', '1K'), - ('3168x1344', '21:9', '2K'), - ('6336x2688', '21:9', '4K'), + ("512x512", "1:1", "512"), + ("1024x1024", "1:1", "1K"), + ("2048x2048", "1:1", "2K"), + ("4096x4096", "1:1", "4K"), + ("256x1024", "1:4", "512"), + ("512x2048", "1:4", "1K"), + ("1024x4096", "1:4", "2K"), + ("2048x8192", "1:4", "4K"), + ("192x1536", "1:8", "512"), + ("384x3072", "1:8", "1K"), + ("768x6144", "1:8", "2K"), + ("1536x12288", "1:8", "4K"), + ("424x632", "2:3", "512"), + ("848x1264", "2:3", "1K"), + ("1696x2528", "2:3", "2K"), + ("3392x5056", "2:3", "4K"), + ("632x424", "3:2", "512"), + ("1264x848", "3:2", "1K"), + ("2528x1696", "3:2", "2K"), + ("5056x3392", "3:2", "4K"), + ("448x600", "3:4", "512"), + ("896x1200", "3:4", "1K"), + ("1792x2400", "3:4", "2K"), + ("3584x4800", "3:4", "4K"), + ("1024x256", "4:1", "512"), + ("2048x512", "4:1", "1K"), + ("4096x1024", "4:1", "2K"), + ("8192x2048", "4:1", "4K"), + ("600x448", "4:3", "512"), + ("1200x896", "4:3", "1K"), + ("2400x1792", "4:3", "2K"), + ("4800x3584", "4:3", "4K"), + ("464x576", "4:5", "512"), + ("928x1152", "4:5", "1K"), + ("1856x2304", "4:5", "2K"), + ("3712x4608", "4:5", "4K"), + ("576x464", "5:4", "512"), + ("1152x928", "5:4", "1K"), + ("2304x1856", "5:4", "2K"), + ("4608x3712", "5:4", "4K"), + ("1536x192", "8:1", "512"), + ("3072x384", "8:1", "1K"), + ("6144x768", "8:1", "2K"), + ("12288x1536", "8:1", "4K"), + ("384x688", "9:16", "512"), + ("768x1376", "9:16", "1K"), + ("1536x2752", "9:16", "2K"), + ("3072x5504", "9:16", "4K"), + ("688x384", "16:9", "512"), + ("1376x768", "16:9", "1K"), + ("2752x1536", "16:9", "2K"), + ("5504x3072", "16:9", "4K"), + ("792x336", "21:9", "512"), + ("1584x672", "21:9", "1K"), + ("3168x1344", "21:9", "2K"), + ("6336x2688", "21:9", "4K"), ) @@ -314,9 +316,7 @@ def test_vertex_ai_response_json_schema_preserves_refs_for_gemini_2(): # $defs and $ref should be preserved (not unpacked) assert "response_json_schema" in transformed_request result_schema = transformed_request["response_json_schema"] - assert ( - "$defs" in result_schema - ), "responseJsonSchema should preserve $defs for Gemini 2.0+" + assert "$defs" in result_schema, "responseJsonSchema should preserve $defs for Gemini 2.0+" def test_vertex_ai_get_json_schema_preserves_refs_for_nested_pydantic(): @@ -396,22 +396,14 @@ def test_vertex_ai_response_json_schema_for_gemini_2(): # Types should be lowercase (standard JSON Schema format) assert transformed_request["response_json_schema"]["type"] == "object" - assert ( - transformed_request["response_json_schema"]["properties"]["name"]["type"] - == "string" - ) - assert ( - transformed_request["response_json_schema"]["properties"]["age"]["type"] - == "integer" - ) + assert transformed_request["response_json_schema"]["properties"]["name"]["type"] == "string" + assert transformed_request["response_json_schema"]["properties"]["age"]["type"] == "integer" # Should NOT have propertyOrdering (not needed for responseJsonSchema) assert "propertyOrdering" not in transformed_request["response_json_schema"] # additionalProperties should be preserved (supported by responseJsonSchema) - assert ( - transformed_request["response_json_schema"].get("additionalProperties") == False - ) + assert transformed_request["response_json_schema"].get("additionalProperties") == False def test_vertex_ai_response_schema_for_old_models(): @@ -648,7 +640,9 @@ def test_response_has_search_grounding_detection(): is False ) assert ( - VertexGeminiConfig._response_has_search_grounding({"candidates": [{"groundingMetadata": {"webSearchQueries": []}}]}) + VertexGeminiConfig._response_has_search_grounding( + {"candidates": [{"groundingMetadata": {"webSearchQueries": []}}]} + ) is False ) assert VertexGeminiConfig._response_has_search_grounding({"candidates": []}) is False @@ -789,9 +783,7 @@ def test_vertex_ai_url_context_tool_use_tokens_billed_as_input_tokens(): "candidates": [ { "urlContextMetadata": {"urlMetadata": []}, - "groundingMetadata": { - "groundingChunks": [{"web": {"uri": "https://example.com", "title": "Example"}}] - }, + "groundingMetadata": {"groundingChunks": [{"web": {"uri": "https://example.com", "title": "Example"}}]}, } ], "usageMetadata": UsageMetadata( @@ -823,9 +815,7 @@ def test_url_context_tool_use_tokens_are_priced_as_input(): "candidates": [ { "urlContextMetadata": {"urlMetadata": []}, - "groundingMetadata": { - "groundingChunks": [{"web": {"uri": "https://example.com", "title": "Example"}}] - }, + "groundingMetadata": {"groundingChunks": [{"web": {"uri": "https://example.com", "title": "Example"}}]}, } ], "usageMetadata": UsageMetadata( @@ -886,9 +876,7 @@ def test_missing_prompt_tokens_details_still_prices_the_full_prompt(): "candidates": [ { "urlContextMetadata": {"urlMetadata": []}, - "groundingMetadata": { - "groundingChunks": [{"web": {"uri": "https://example.com", "title": "Example"}}] - }, + "groundingMetadata": {"groundingChunks": [{"web": {"uri": "https://example.com", "title": "Example"}}]}, } ], "usageMetadata": UsageMetadata( @@ -925,9 +913,7 @@ def test_streaming_chunk_includes_reasoning_tokens(): "thoughtsTokenCount": 3, }, } - iterator = ModelResponseIterator( - streaming_response=[], sync_stream=True, logging_obj=litellm_logging - ) + iterator = ModelResponseIterator(streaming_response=[], sync_stream=True, logging_obj=litellm_logging) streaming_chunk = iterator.chunk_parser(chunk) assert streaming_chunk.usage is not None assert streaming_chunk.usage.prompt_tokens == 5 @@ -963,17 +949,12 @@ def test_streaming_chunk_includes_reasoning_content(): "usageMetadata": {}, } - iterator = ModelResponseIterator( - streaming_response=[], sync_stream=True, logging_obj=litellm_logging - ) + iterator = ModelResponseIterator(streaming_response=[], sync_stream=True, logging_obj=litellm_logging) streaming_chunk = iterator.chunk_parser(chunk) # The text content should be empty and reasoning_content should be populated assert streaming_chunk.choices[0].delta.content is None - assert ( - streaming_chunk.choices[0].delta.reasoning_content - == "I'm thinking through the problem..." - ) + assert streaming_chunk.choices[0].delta.reasoning_content == "I'm thinking through the problem..." def test_streaming_chunk_with_tool_calls_and_thought_includes_reasoning_content(): @@ -1018,24 +999,16 @@ def test_streaming_chunk_with_tool_calls_and_thought_includes_reasoning_content( }, } - iterator = ModelResponseIterator( - streaming_response=[], sync_stream=True, logging_obj=litellm_logging - ) + iterator = ModelResponseIterator(streaming_response=[], sync_stream=True, logging_obj=litellm_logging) streaming_chunk = iterator.chunk_parser(chunk) # Verify reasoning_content comes from the thought: true part - assert ( - streaming_chunk.choices[0].delta.reasoning_content - == "Let me think about how to get the time..." - ) + assert streaming_chunk.choices[0].delta.reasoning_content == "Let me think about how to get the time..." # Verify tool calls are also present assert streaming_chunk.choices[0].delta.tool_calls is not None assert len(streaming_chunk.choices[0].delta.tool_calls) == 1 - assert ( - streaming_chunk.choices[0].delta.tool_calls[0].function.name - == "get_current_time" - ) + assert streaming_chunk.choices[0].delta.tool_calls[0].function.name == "get_current_time" def test_streaming_chunk_with_tool_calls_no_thought_no_reasoning_content(): @@ -1077,9 +1050,7 @@ def test_streaming_chunk_with_tool_calls_no_thought_no_reasoning_content(): }, } - iterator = ModelResponseIterator( - streaming_response=[], sync_stream=True, logging_obj=litellm_logging - ) + iterator = ModelResponseIterator(streaming_response=[], sync_stream=True, logging_obj=litellm_logging) streaming_chunk = iterator.chunk_parser(chunk) # reasoning_content should be None - thoughtSignature alone does NOT mean reasoning @@ -1286,11 +1257,7 @@ def test_vertex_ai_usage_metadata_with_image_tokens_in_prompt(): # Verify the math: prompt_tokens = text + image # 533 = 6 (text) + 527 (image) - assert ( - result.prompt_tokens_details.text_tokens - + result.prompt_tokens_details.image_tokens - == result.prompt_tokens - ) + assert result.prompt_tokens_details.text_tokens + result.prompt_tokens_details.image_tokens == result.prompt_tokens def test_map_response_modalities_video(): @@ -1480,9 +1447,7 @@ def test_vertex_ai_streaming_usage_calculation(): } # Create iterator and parse chunk - iterator = ModelResponseIterator( - streaming_response=[], sync_stream=True, logging_obj=MagicMock() - ) + iterator = ModelResponseIterator(streaming_response=[], sync_stream=True, logging_obj=MagicMock()) iterator.chunk_parser(chunk) # Verify calculate_usage was called with correct parameters @@ -1554,9 +1519,7 @@ def test_vertex_ai_streaming_usage_web_search_calculation(): } # Create iterator and parse chunk - iterator = ModelResponseIterator( - streaming_response=[], sync_stream=True, logging_obj=MagicMock() - ) + iterator = ModelResponseIterator(streaming_response=[], sync_stream=True, logging_obj=MagicMock()) completed_response = iterator.chunk_parser(chunk) usage: Usage = completed_response.usage @@ -1775,7 +1738,9 @@ async def test_gemini_image_generation_async_stream(respx_mock: respx.MockRouter "modelVersion": "gemini-2.5-flash-image", }, { - "candidates": [{"content": {"role": "model", "parts": [{"text": ""}]}, "finishReason": "STOP", "index": 0}], + "candidates": [ + {"content": {"role": "model", "parts": [{"text": ""}]}, "finishReason": "STOP", "index": 0} + ], "usageMetadata": {"promptTokenCount": 13, "candidatesTokenCount": 1290, "totalTokenCount": 1303}, "modelVersion": "gemini-2.5-flash-image", }, @@ -2029,9 +1994,14 @@ def test_gemini_openai_web_search_tool_to_google_search(respx_mock: respx.MockRo GEMINI_GROUNDING_METADATA: Final = { "webSearchQueries": ["capital of France"], - "searchEntryPoint": {"renderedContent": "
"}, + "searchEntryPoint": {"renderedContent": '
'}, "groundingChunks": [ - {"web": {"uri": "https://vertexaisearch.cloud.google.com/grounding-api-redirect/AbF9w", "title": "wikipedia.org"}} + { + "web": { + "uri": "https://vertexaisearch.cloud.google.com/grounding-api-redirect/AbF9w", + "title": "wikipedia.org", + } + } ], "groundingSupports": [ { @@ -2188,9 +2158,7 @@ def test_vertex_ai_transform_parts(): # Test case 2: Tool call mode (is_function_call=False) - Single message with multiple tool calls parts_with_multiple_functions = [ - HttpxPartType( - functionCall={"name": "get_current_weather", "args": {"location": "Boston"}} - ), + HttpxPartType(functionCall={"name": "get_current_weather", "args": {"location": "Boston"}}), HttpxPartType( functionCall={ "name": "get_forecast", @@ -2220,9 +2188,7 @@ def test_vertex_ai_transform_parts(): # Test case 3: Simulating multiple messages - cumulative indexing across messages # First message with 2 tool calls (starting from index 0) first_message_parts = [ - HttpxPartType( - functionCall={"name": "get_weather", "args": {"location": "Boston"}} - ), + HttpxPartType(functionCall={"name": "get_weather", "args": {"location": "Boston"}}), HttpxPartType(functionCall={"name": "get_time", "args": {"timezone": "EST"}}), ] @@ -2239,9 +2205,7 @@ def test_vertex_ai_transform_parts(): # Second message with 1 tool call (continuing from previous index) second_message_parts = [ - HttpxPartType( - functionCall={"name": "send_email", "args": {"to": "user@example.com"}} - ), + HttpxPartType(functionCall={"name": "send_email", "args": {"to": "user@example.com"}}), ] function, tools, updated_idx = VertexGeminiConfig._transform_parts( @@ -2259,9 +2223,7 @@ def test_vertex_ai_transform_parts(): # Third message with 2 more tool calls (continuing from previous index) third_message_parts = [ - HttpxPartType( - functionCall={"name": "create_calendar_event", "args": {"title": "Meeting"}} - ), + HttpxPartType(functionCall={"name": "create_calendar_event", "args": {"title": "Meeting"}}), HttpxPartType(functionCall={"name": "set_reminder", "args": {"time": "10:00"}}), ] @@ -2308,9 +2270,7 @@ def test_vertex_ai_transform_parts(): assert updated_idx == 10 # Index should remain unchanged # Test case 6: Function call with empty args - parts_with_empty_args = [ - HttpxPartType(functionCall={"name": "simple_function", "args": {}}) - ] + parts_with_empty_args = [HttpxPartType(functionCall={"name": "simple_function", "args": {}})] function, tools, updated_idx = VertexGeminiConfig._transform_parts( parts=parts_with_empty_args, cumulative_tool_call_idx=0, is_function_call=True @@ -2326,13 +2286,9 @@ def test_vertex_ai_transform_parts(): # Test case 7: Mixed content with function calls - ensuring tool call IDs are unique mixed_parts = [ HttpxPartType(text="Before function call"), - HttpxPartType( - functionCall={"name": "function_a", "args": {"param": "value_a"}} - ), + HttpxPartType(functionCall={"name": "function_a", "args": {"param": "value_a"}}), HttpxPartType(text="Between function calls"), - HttpxPartType( - functionCall={"name": "function_b", "args": {"param": "value_b"}} - ), + HttpxPartType(functionCall={"name": "function_b", "args": {"param": "value_b"}}), HttpxPartType(text="After function calls"), ] @@ -2524,25 +2480,17 @@ def test_vertex_ai_tool_call_id_format(): tool_id = tool["id"] # Should start with 'call_' - assert tool_id.startswith( - "call_" - ), f"ID should start with 'call_', got: {tool_id}" + assert tool_id.startswith("call_"), f"ID should start with 'call_', got: {tool_id}" # Should have exactly 33 total characters (call_ + 28 hex chars) - assert ( - len(tool_id) == 33 - ), f"ID should be 33 characters long, got {len(tool_id)}: {tool_id}" + assert len(tool_id) == 33, f"ID should be 33 characters long, got {len(tool_id)}: {tool_id}" # The part after 'call_' should be 28 hex characters hex_part = tool_id[5:] # Remove 'call_' prefix - assert ( - len(hex_part) == 28 - ), f"Hex part should be 28 characters, got {len(hex_part)}: {hex_part}" + assert len(hex_part) == 28, f"Hex part should be 28 characters, got {len(hex_part)}: {hex_part}" # Should only contain valid hex characters - assert re.match( - r"^[0-9a-f]{28}$", hex_part - ), f"Should contain only lowercase hex chars, got: {hex_part}" + assert re.match(r"^[0-9a-f]{28}$", hex_part), f"Should contain only lowercase hex chars, got: {hex_part}" # Verify IDs are unique assert tools[0]["id"] != tools[1]["id"], "Tool call IDs should be unique" @@ -2559,9 +2507,7 @@ def test_vertex_ai_tool_call_id_format(): ids_generated.add(test_tools[0]["id"]) # All generated IDs should be unique - assert ( - len(ids_generated) == 10 - ), f"All 10 IDs should be unique, got {len(ids_generated)} unique IDs" + assert len(ids_generated) == 10, f"All 10 IDs should be unique, got {len(ids_generated)} unique IDs" def test_vertex_ai_map_google_maps_tool_simple(): @@ -2675,19 +2621,16 @@ def test_vertex_ai_penalty_parameters_validation(): for model, should_support in test_cases: # Test _supports_penalty_parameters method - assert ( - v._supports_penalty_parameters(model) == should_support - ), f"Model {model} penalty support should be {should_support}" + assert v._supports_penalty_parameters(model) == should_support, ( + f"Model {model} penalty support should be {should_support}" + ) # Test get_supported_openai_params method supported_params = v.get_supported_openai_params(model) - has_penalty_params = ( - "frequency_penalty" in supported_params - and "presence_penalty" in supported_params + has_penalty_params = "frequency_penalty" in supported_params and "presence_penalty" in supported_params + assert has_penalty_params == should_support, ( + f"Model {model} should {'include' if should_support else 'exclude'} penalty params in supported list" ) - assert ( - has_penalty_params == should_support - ), f"Model {model} should {'include' if should_support else 'exclude'} penalty params in supported list" # Test parameter mapping for unsupported model model = "gemini-2.5-pro-preview-06-05" @@ -2707,12 +2650,8 @@ def test_vertex_ai_penalty_parameters_validation(): ) # Penalty parameters should be filtered out for unsupported models - assert ( - "frequency_penalty" not in result - ), "frequency_penalty should be filtered out for unsupported model" - assert ( - "presence_penalty" not in result - ), "presence_penalty should be filtered out for unsupported model" + assert "frequency_penalty" not in result, "frequency_penalty should be filtered out for unsupported model" + assert "presence_penalty" not in result, "presence_penalty should be filtered out for unsupported model" # Other parameters should still be included assert "temperature" in result, "temperature should still be included" @@ -2741,18 +2680,18 @@ def test_vertex_ai_gemini_3_penalty_parameters_unsupported(): for model in gemini_3_models: # Test _supports_penalty_parameters method - assert ( - v._supports_penalty_parameters(model) == False - ), f"Gemini 3 model {model} should not support penalty parameters" + assert v._supports_penalty_parameters(model) == False, ( + f"Gemini 3 model {model} should not support penalty parameters" + ) # Test get_supported_openai_params method supported_params = v.get_supported_openai_params(model) - assert ( - "frequency_penalty" not in supported_params - ), f"frequency_penalty should not be in supported params for {model}" - assert ( - "presence_penalty" not in supported_params - ), f"presence_penalty should not be in supported params for {model}" + assert "frequency_penalty" not in supported_params, ( + f"frequency_penalty should not be in supported params for {model}" + ) + assert "presence_penalty" not in supported_params, ( + f"presence_penalty should not be in supported params for {model}" + ) # Test parameter mapping - penalty params should be filtered out non_default_params = { @@ -2771,36 +2710,28 @@ def test_vertex_ai_gemini_3_penalty_parameters_unsupported(): ) # Penalty parameters should be filtered out for Gemini 3 models - assert ( - "frequency_penalty" not in result - ), f"frequency_penalty should be filtered out for Gemini 3 model {model}" - assert ( - "presence_penalty" not in result - ), f"presence_penalty should be filtered out for Gemini 3 model {model}" + assert "frequency_penalty" not in result, f"frequency_penalty should be filtered out for Gemini 3 model {model}" + assert "presence_penalty" not in result, f"presence_penalty should be filtered out for Gemini 3 model {model}" # Other parameters should still be included - assert ( - "temperature" in result - ), f"temperature should still be included for Gemini 3 model {model}" - assert ( - "max_output_tokens" in result - ), f"max_output_tokens should still be included for Gemini 3 model {model}" + assert "temperature" in result, f"temperature should still be included for Gemini 3 model {model}" + assert "max_output_tokens" in result, f"max_output_tokens should still be included for Gemini 3 model {model}" assert result["temperature"] == 0.7 assert result["max_output_tokens"] == 100 # Test that non-Gemini 3 models still support penalty parameters (if they're not in the unsupported list) non_gemini_3_model = "gemini-2.5-pro" - assert ( - v._supports_penalty_parameters(non_gemini_3_model) == True - ), f"Non-Gemini 3 model {non_gemini_3_model} should support penalty parameters" + assert v._supports_penalty_parameters(non_gemini_3_model) == True, ( + f"Non-Gemini 3 model {non_gemini_3_model} should support penalty parameters" + ) supported_params = v.get_supported_openai_params(non_gemini_3_model) - assert ( - "frequency_penalty" in supported_params - ), f"frequency_penalty should be in supported params for {non_gemini_3_model}" - assert ( - "presence_penalty" in supported_params - ), f"presence_penalty should be in supported params for {non_gemini_3_model}" + assert "frequency_penalty" in supported_params, ( + f"frequency_penalty should be in supported params for {non_gemini_3_model}" + ) + assert "presence_penalty" in supported_params, ( + f"presence_penalty should be in supported params for {non_gemini_3_model}" + ) def test_vertex_ai_annotation_streaming_events(): @@ -2823,14 +2754,10 @@ def test_vertex_ai_annotation_streaming_events(): chunk_with_annotations = { "candidates": [ { - "content": { - "parts": [{"text": "The weather in San Francisco today is clear."}] - }, + "content": {"parts": [{"text": "The weather in San Francisco today is clear."}]}, "groundingMetadata": { "webSearchQueries": ["weather San Francisco today"], - "searchEntryPoint": { - "renderedContent": "
Search results
" - }, + "searchEntryPoint": {"renderedContent": "
Search results
"}, "groundingChunks": [ { "web": { @@ -2862,9 +2789,7 @@ def test_vertex_ai_annotation_streaming_events(): } # Create iterator and parse chunk - iterator = ModelResponseIterator( - streaming_response=[], sync_stream=True, logging_obj=litellm_logging - ) + iterator = ModelResponseIterator(streaming_response=[], sync_stream=True, logging_obj=litellm_logging) streaming_chunk = iterator.chunk_parser(chunk_with_annotations) # Verify the chunk was parsed correctly @@ -2986,12 +2911,8 @@ def test_vertex_ai_annotation_conversion(): } # Convert grounding metadata to annotations - content_text = ( - "The weather in San Francisco is currently 72°F and the time is 2:30 PM" - ) - annotations = VertexGeminiConfig._convert_grounding_metadata_to_annotations( - [grounding_metadata], content_text - ) + content_text = "The weather in San Francisco is currently 72°F and the time is 2:30 PM" + annotations = VertexGeminiConfig._convert_grounding_metadata_to_annotations([grounding_metadata], content_text) # Verify annotations were created assert len(annotations) == 3 # One for each grounding support @@ -3033,9 +2954,7 @@ def test_vertex_ai_annotation_empty_grounding_metadata(): # Test with empty grounding metadata empty_metadata = {} - annotations = VertexGeminiConfig._convert_grounding_metadata_to_annotations( - [empty_metadata], "test content" - ) + annotations = VertexGeminiConfig._convert_grounding_metadata_to_annotations([empty_metadata], "test content") assert len(annotations) == 0 # Test with missing groundingSupports @@ -3043,9 +2962,7 @@ def test_vertex_ai_annotation_empty_grounding_metadata(): "webSearchQueries": ["test query"], "groundingChunks": [{"web": {"uri": "https://example.com", "title": "Test"}}], } - annotations = VertexGeminiConfig._convert_grounding_metadata_to_annotations( - [metadata_no_supports], "test content" - ) + annotations = VertexGeminiConfig._convert_grounding_metadata_to_annotations([metadata_no_supports], "test content") assert len(annotations) == 0 # Test with empty groundingSupports @@ -3408,9 +3325,7 @@ def test_reasoning_effort_maps_to_thinking_level_gemini_3(): def test_gemini_37_38_flash_floor_minimal_thinking_level( local_model_cost_map, model, reasoning_effort, include_thoughts ): - result = VertexGeminiConfig._map_reasoning_effort_to_thinking_level( - reasoning_effort, model - ) + result = VertexGeminiConfig._map_reasoning_effort_to_thinking_level(reasoning_effort, model) assert result["thinkingLevel"] == "low" assert result["includeThoughts"] is include_thoughts @@ -3434,9 +3349,7 @@ def test_gemini_37_38_flash_floor_minimal_thinking_level( def test_gemini_flash_minimal_thinking_support( local_model_cost_map, model, reasoning_effort, expected_level, include_thoughts ): - result = VertexGeminiConfig._map_reasoning_effort_to_thinking_level( - reasoning_effort, model - ) + result = VertexGeminiConfig._map_reasoning_effort_to_thinking_level(reasoning_effort, model) assert result["thinkingLevel"] == expected_level assert result["includeThoughts"] is include_thoughts @@ -3446,12 +3359,8 @@ def test_gemini_38_flash_feature_flag_uses_low_thinking_level(local_model_cost_m monkeypatch.setattr(litellm, "enable_gemini_default_thinking_level_low", True) thinking_param = {"type": "enabled", "budget_tokens": 1024} - result_38 = VertexGeminiConfig._map_thinking_param( - thinking_param, model="gemini-3.8-flash" - ) - result_36 = VertexGeminiConfig._map_thinking_param( - thinking_param, model="gemini-3.6-flash" - ) + result_38 = VertexGeminiConfig._map_thinking_param(thinking_param, model="gemini-3.8-flash") + result_36 = VertexGeminiConfig._map_thinking_param(thinking_param, model="gemini-3.6-flash") assert result_38["thinkingLevel"] == "low" assert result_36["thinkingLevel"] == "minimal" @@ -3933,9 +3842,7 @@ def test_gemini_image_models_excluded_from_thinking(): ) # None of these should have thinkingConfig - assert ( - "thinkingConfig" not in result - ), f"Model {model} should not have thinkingConfig" + assert "thinkingConfig" not in result, f"Model {model} should not have thinkingConfig" def test_partial_json_chunk_after_first_chunk(): @@ -3966,9 +3873,7 @@ def test_partial_json_chunk_after_first_chunk(): first_chunk = '{"candidates": [{"content": {"parts": [{"text": "Hello"}]}}]}' result1 = iterator.handle_valid_json_chunk(first_chunk) assert result1 is not None, "First complete chunk should parse OK" - assert ( - iterator.sent_first_chunk is True - ), "sent_first_chunk should be True after first chunk" + assert iterator.sent_first_chunk is True, "sent_first_chunk should be True after first chunk" # Later chunk arrives PARTIAL (simulating network fragmentation) partial_chunk = '{"candidates": [{"content":' @@ -3976,9 +3881,7 @@ def test_partial_json_chunk_after_first_chunk(): # Should switch to accumulation mode instead of crashing assert result2 is None, "Partial chunk should return None while accumulating" - assert ( - iterator.chunk_type == "accumulated_json" - ), "Should switch to accumulated_json mode" + assert iterator.chunk_type == "accumulated_json", "Should switch to accumulated_json mode" def test_partial_json_chunk_on_first_chunk(): @@ -3998,9 +3901,7 @@ def test_partial_json_chunk_on_first_chunk(): result = iterator.handle_valid_json_chunk(partial) assert result is None, "Partial first chunk should return None" - assert ( - iterator.chunk_type == "accumulated_json" - ), "Should switch to accumulated_json mode" + assert iterator.chunk_type == "accumulated_json", "Should switch to accumulated_json mode" def test_accumulated_json_does_not_reparse_every_fragment(): @@ -4032,16 +3933,12 @@ def test_accumulated_json_does_not_reparse_every_fragment(): iterator.chunk_type = "accumulated_json" text = "x" * 200_000 # no braces/brackets so only the final fragment closes - blob = json.dumps( - {"candidates": [{"content": {"role": "model", "parts": [{"text": text}]}}]} - ) + blob = json.dumps({"candidates": [{"content": {"role": "model", "parts": [{"text": text}]}}]}) fragments = [blob[i : i + 4096] for i in range(0, len(blob), 4096)] assert len(fragments) > 10, "need a multi-fragment payload to exercise the bug" parsed = None - with patch.object( - json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode - ) as spy: + with patch.object(json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode) as spy: for fragment in fragments: out = iterator.handle_accumulated_json_chunk(chunk=fragment) if out is not None: @@ -4073,9 +3970,7 @@ def test_accumulated_json_partial_fragment_returns_none_without_parsing(): ) iterator.chunk_type = "accumulated_json" - with patch.object( - json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode - ) as spy: + with patch.object(json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode) as spy: result = iterator.handle_accumulated_json_chunk( chunk='{"candidates": [{"content": {"parts": [{"text": "partial' ) @@ -4174,17 +4069,11 @@ def test_vertex_ai_multiple_tool_types_separate_objects(): tool_types_in_first = [k for k in tools[0].keys()] tool_types_in_second = [k for k in tools[1].keys()] - assert ( - len(tool_types_in_first) == 1 - ), f"First Tool should have exactly 1 type, got {tool_types_in_first}" - assert ( - len(tool_types_in_second) == 1 - ), f"Second Tool should have exactly 1 type, got {tool_types_in_second}" + assert len(tool_types_in_first) == 1, f"First Tool should have exactly 1 type, got {tool_types_in_first}" + assert len(tool_types_in_second) == 1, f"Second Tool should have exactly 1 type, got {tool_types_in_second}" # Verify the correct tool types are present - assert ( - "enterpriseWebSearch" in tools[0] - ), "First Tool should contain enterpriseWebSearch" + assert "enterpriseWebSearch" in tools[0], "First Tool should contain enterpriseWebSearch" assert "url_context" in tools[1], "Second Tool should contain url_context" @@ -4646,9 +4535,7 @@ def test_vertex_ai_multiple_function_declarations_grouped(): ) # Should have only 1 Tool object (function declarations grouped) - assert ( - len(tools) == 1 - ), f"Expected 1 Tool object for grouped functions, got {len(tools)}" + assert len(tools) == 1, f"Expected 1 Tool object for grouped functions, got {len(tools)}" # Should contain function_declarations with 2 functions assert "function_declarations" in tools[0] @@ -5070,9 +4957,7 @@ def test_chunk_parser_handles_prompt_feedback_block(): logging_obj = Mock() logging_obj.optional_params = {} - streaming_obj = ModelResponseIterator( - streaming_response=iter([]), sync_stream=True, logging_obj=logging_obj - ) + streaming_obj = ModelResponseIterator(streaming_response=iter([]), sync_stream=True, logging_obj=logging_obj) # Act result = streaming_obj.chunk_parser(blocked_chunk) @@ -5080,9 +4965,9 @@ def test_chunk_parser_handles_prompt_feedback_block(): # Assert assert result is not None, "Result should not be None" assert len(result.choices) == 1, "Should have exactly one choice" - assert ( - result.choices[0].finish_reason == "content_filter" - ), f"finish_reason should be content_filter, got {result.choices[0].finish_reason}" + assert result.choices[0].finish_reason == "content_filter", ( + f"finish_reason should be content_filter, got {result.choices[0].finish_reason}" + ) assert result.choices[0].delta.content is None, "content should be None" @@ -5106,9 +4991,7 @@ def test_chunk_parser_handles_prompt_feedback_safety_block(): logging_obj = Mock() logging_obj.optional_params = {} - streaming_obj = ModelResponseIterator( - streaming_response=iter([]), sync_stream=True, logging_obj=logging_obj - ) + streaming_obj = ModelResponseIterator(streaming_response=iter([]), sync_stream=True, logging_obj=logging_obj) # Act result = streaming_obj.chunk_parser(blocked_chunk) @@ -5145,9 +5028,7 @@ def test_chunk_parser_handles_prompt_feedback_block_with_usage(): logging_obj = Mock() logging_obj.optional_params = {} - streaming_obj = ModelResponseIterator( - streaming_response=iter([]), sync_stream=True, logging_obj=logging_obj - ) + streaming_obj = ModelResponseIterator(streaming_response=iter([]), sync_stream=True, logging_obj=logging_obj) # Act result = streaming_obj.chunk_parser(blocked_chunk) @@ -5155,23 +5036,17 @@ def test_chunk_parser_handles_prompt_feedback_block_with_usage(): # Assert - 验证 content_filter 响应和 usage 都被正确处理 assert result is not None, "Result should not be None" assert len(result.choices) == 1, "Should have exactly one choice" - assert ( - result.choices[0].finish_reason == "content_filter" - ), f"finish_reason should be content_filter, got {result.choices[0].finish_reason}" + assert result.choices[0].finish_reason == "content_filter", ( + f"finish_reason should be content_filter, got {result.choices[0].finish_reason}" + ) assert result.choices[0].delta.content is None, "content should be None" # 验证 usage 信息被正确提取 assert hasattr(result, "usage"), "result should have usage attribute" assert result.usage is not None, "usage should not be None" - assert ( - result.usage.prompt_tokens == 8175 - ), f"prompt_tokens should be 8175, got {result.usage.prompt_tokens}" - assert ( - result.usage.completion_tokens == 0 - ), f"completion_tokens should be 0, got {result.usage.completion_tokens}" - assert ( - result.usage.total_tokens == 8175 - ), f"total_tokens should be 8175, got {result.usage.total_tokens}" + assert result.usage.prompt_tokens == 8175, f"prompt_tokens should be 8175, got {result.usage.prompt_tokens}" + assert result.usage.completion_tokens == 0, f"completion_tokens should be 0, got {result.usage.completion_tokens}" + assert result.usage.total_tokens == 8175, f"total_tokens should be 8175, got {result.usage.total_tokens}" def test_vertex_ai_traffic_type_preserved_in_hidden_params_streaming(): @@ -5190,14 +5065,10 @@ def test_vertex_ai_traffic_type_preserved_in_hidden_params_streaming(): }, } - iterator = ModelResponseIterator( - streaming_response=[], sync_stream=True, logging_obj=MagicMock() - ) + iterator = ModelResponseIterator(streaming_response=[], sync_stream=True, logging_obj=MagicMock()) result = iterator.chunk_parser(chunk) - assert ( - result._hidden_params["provider_specific_fields"]["traffic_type"] == "ON_DEMAND" - ) + assert result._hidden_params["provider_specific_fields"]["traffic_type"] == "ON_DEMAND" def test_vertex_ai_traffic_type_preserved_in_hidden_params_non_streaming(): @@ -5236,10 +5107,7 @@ def test_vertex_ai_traffic_type_preserved_in_hidden_params_non_streaming(): encoding=None, ) - assert ( - result._hidden_params["provider_specific_fields"]["traffic_type"] - == "PROVISIONED_THROUGHPUT" - ) + assert result._hidden_params["provider_specific_fields"]["traffic_type"] == "PROVISIONED_THROUGHPUT" def test_vertex_ai_service_tier_streaming(): @@ -5323,9 +5191,7 @@ def test_vertex_ai_traffic_type_surfaced_in_responses_api(): from litellm.types.utils import Choices, Message model_response = ModelResponse() - model_response._hidden_params["provider_specific_fields"] = { - "traffic_type": "ON_DEMAND" - } + model_response._hidden_params["provider_specific_fields"] = {"traffic_type": "ON_DEMAND"} model_response.choices = [ Choices( message=Message(content="Hello", role="assistant"), @@ -5334,15 +5200,15 @@ def test_vertex_ai_traffic_type_surfaced_in_responses_api(): ) ] - responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( - request_input="test", - chat_completion_response=model_response, - responses_api_request={}, + responses_api_response = ( + LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="test", + chat_completion_response=model_response, + responses_api_request={}, + ) ) - assert ( - responses_api_response.provider_specific_fields["traffic_type"] == "ON_DEMAND" - ) + assert responses_api_response.provider_specific_fields["traffic_type"] == "ON_DEMAND" def test_vertex_ai_web_search_options_parameter(): @@ -5416,12 +5282,8 @@ def test_vertex_ai_web_search_options_in_map_openai_params(): assert "tools" in optional_params, "tools should be added to optional_params" assert len(optional_params["tools"]) == 1, "Should have exactly one tool" assert "googleSearch" in optional_params["tools"][0], "Tool should be googleSearch" - assert ( - optional_params["tools"][0]["googleSearch"] == {} - ), "googleSearch should be empty config" - assert ( - "web_search_options" not in optional_params - ), "web_search_options should be removed after transformation" + assert optional_params["tools"][0]["googleSearch"] == {}, "googleSearch should be empty config" + assert "web_search_options" not in optional_params, "web_search_options should be removed after transformation" def test_vertex_ai_service_tier_in_map_openai_params(): @@ -5509,24 +5371,16 @@ def test_vertex_ai_usage_metadata_with_video_tokens_in_prompt(): # Verify prompt token details include video tokens assert result.prompt_tokens_details is not None - assert ( - result.prompt_tokens_details.video_tokens == 10240 - ), "Prompt video tokens should be 10240" - assert ( - result.prompt_tokens_details.text_tokens == 9 - ), "Prompt text tokens should be 9" - assert ( - result.prompt_tokens_details.audio_tokens == 200 - ), "Prompt audio tokens should be 200" + assert result.prompt_tokens_details.video_tokens == 10240, "Prompt video tokens should be 10240" + assert result.prompt_tokens_details.text_tokens == 9, "Prompt text tokens should be 9" + assert result.prompt_tokens_details.audio_tokens == 200, "Prompt audio tokens should be 200" # Verify completion token details assert result.completion_tokens_details is not None - assert ( - result.completion_tokens_details.text_tokens == 79 - ), "Completion text tokens should be 79" - assert ( - result.completion_tokens_details.video_tokens is None - ), "Completion video tokens should be None (text-only response)" + assert result.completion_tokens_details.text_tokens == 79, "Completion text tokens should be 79" + assert result.completion_tokens_details.video_tokens is None, ( + "Completion video tokens should be None (text-only response)" + ) def test_vertex_ai_usage_metadata_with_video_tokens_in_candidates(): @@ -5667,9 +5521,9 @@ def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt(): # DOCUMENT tokens should be included in text_tokens: 8 (TEXT) + 774 (DOCUMENT) = 782 assert result.prompt_tokens_details is not None - assert ( - result.prompt_tokens_details.text_tokens == 782 - ), "DOCUMENT modality tokens should be added to text_tokens (8 TEXT + 774 DOCUMENT = 782)" + assert result.prompt_tokens_details.text_tokens == 782, ( + "DOCUMENT modality tokens should be added to text_tokens (8 TEXT + 774 DOCUMENT = 782)" + ) # Verify completion token details assert result.completion_tokens_details is not None @@ -6166,9 +6020,7 @@ def test_mid_stream_429_error_raises_during_iteration(): { "content": { "role": "model", - "parts": [ - {"text": "Let me think about this...", "thought": True} - ], + "parts": [{"text": "Let me think about this...", "thought": True}], }, "index": 0, } @@ -6188,9 +6040,7 @@ def test_mid_stream_429_error_raises_during_iteration(): { "content": { "role": "model", - "parts": [ - {"text": "I'll generate the image now.", "thought": True} - ], + "parts": [{"text": "I'll generate the image now.", "thought": True}], }, "index": 0, } @@ -6227,6 +6077,7 @@ def test_mid_stream_429_error_raises_during_iteration(): # Iterate the stream: first chunks should succeed, then 429 error should be raised results = [] + def _drain(): for chunk in streaming_obj: if chunk is not None: @@ -6236,9 +6087,7 @@ def test_mid_stream_429_error_raises_during_iteration(): _drain() # Verify: received normal chunks before the error - assert ( - len(results) >= 1 - ), "Should have received at least 1 normal chunk before the error" + assert len(results) >= 1, "Should have received at least 1 normal chunk before the error" # Verify: 429 error is properly raised assert exc_info.value.status_code == 429 @@ -6427,9 +6276,7 @@ def _accumulating_gemini_iterator(): ModelResponseIterator, ) - iterator = ModelResponseIterator( - streaming_response=[], sync_stream=True, logging_obj=MagicMock() - ) + iterator = ModelResponseIterator(streaming_response=[], sync_stream=True, logging_obj=MagicMock()) iterator.chunk_type = "accumulated_json" return iterator @@ -6978,6 +6825,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -7030,6 +6878,7 @@ def isolate_litellm_state(): setattr(litellm, attr, original_value) _invalidate_model_cost_lowercase_map() + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), @@ -7050,6 +6899,7 @@ _SCALAR_DEFAULTS = { "api_key": getattr(litellm, "api_key", None), } + @pytest.fixture(scope="module") def setup_and_teardown(): """ @@ -7072,6 +6922,7 @@ def setup_and_teardown(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_thought_true_creates_thinking_block(): """ @@ -7086,6 +6937,7 @@ def test_thought_true_creates_thinking_block(): assert block["thinking"] == "Some thinking" assert block["signature"] == "sig-1" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_thought_true_with_empty_text_creates_block(): """ @@ -7097,6 +6949,7 @@ def test_thought_true_with_empty_text_creates_block(): assert len(thinking_blocks) == 1 assert thinking_blocks[0]["thinking"] == "" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_thought_signature_without_thought_does_not_create_block(): """ @@ -7109,6 +6962,7 @@ def test_thought_signature_without_thought_does_not_create_block(): thinking_blocks = config._extract_thinking_blocks_from_parts(parts) assert thinking_blocks == [] + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_extract_thought_signatures_from_regular_parts(): """ @@ -7128,6 +6982,7 @@ def test_extract_thought_signatures_from_regular_parts(): assert len(signatures) == 1 assert signatures[0] == "sig-regular-123" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_extract_multiple_thought_signatures(): """ @@ -7146,6 +7001,7 @@ def test_extract_multiple_thought_signatures(): assert signatures[0] == "sig-1" assert signatures[1] == "sig-2" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_round_trip_thought_signature_in_conversation(): """ @@ -7180,6 +7036,7 @@ def test_round_trip_thought_signature_in_conversation(): assert "thoughtSignature" in text_part assert text_part["thoughtSignature"] == "sig-round-trip-abc" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_round_trip_without_thought_signature_still_works(): """ @@ -7418,18 +7275,10 @@ def test_gemini_flash_image_preview_models(model_name: str): mock_response = ImageResponse() mock_response.data = [ImageObject(b64_json="test_base64_data", url=None)] - with patch( - "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post" - ) as mock_post: + with patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") as mock_post: mock_http_response = MagicMock() mock_http_response.json.return_value = { - "candidates": [ - { - "content": { - "parts": [{"inlineData": {"data": "test_base64_image_data"}}] - } - } - ] + "candidates": [{"content": {"parts": [{"inlineData": {"data": "test_base64_image_data"}}]}}] } mock_http_response.status_code = 200 mock_post.return_value = mock_http_response @@ -7447,9 +7296,7 @@ def test_gemini_flash_image_preview_models(model_name: str): mock_post.assert_called_once() call_args = mock_post.call_args - called_url = ( - call_args[0][0] if call_args[0] else call_args.kwargs.get("url", "") - ) + called_url = call_args[0][0] if call_args[0] else call_args.kwargs.get("url", "") assert ":generateContent" in called_url assert model_name.split("/", 1)[1] in called_url @@ -7481,23 +7328,13 @@ def test_gemini_flash_image_preview_models(model_name: str): ), ], ) -def test_gemini_image_generation_forwards_image_config( - model: str, kwargs: dict, expected_image_config: dict -): +def test_gemini_image_generation_forwards_image_config(model: str, kwargs: dict, expected_image_config: dict): from unittest.mock import patch, MagicMock - with patch( - "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post" - ) as mock_post: + with patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") as mock_post: mock_http_response = MagicMock() mock_http_response.json.return_value = { - "candidates": [ - { - "content": { - "parts": [{"inlineData": {"data": "test_base64_image_data"}}] - } - } - ] + "candidates": [{"content": {"parts": [{"inlineData": {"data": "test_base64_image_data"}}]}}] } mock_http_response.status_code = 200 mock_post.return_value = mock_http_response @@ -7558,9 +7395,7 @@ def test_gemini_image_generation_openai_size_maps_to_google_table( map_openai_size_to_gemini_image_config, ) - assert map_openai_size_to_gemini_image_config( - size, "gemini-3-pro-image-preview" - ) == { + assert map_openai_size_to_gemini_image_config(size, "gemini-3-pro-image-preview") == { "aspectRatio": expected_aspect_ratio, "imageSize": expected_image_size, } @@ -7584,9 +7419,7 @@ def test_gemini_image_generation_openai_size_snaps_to_nearest_option( map_openai_size_to_gemini_image_config, ) - assert map_openai_size_to_gemini_image_config( - size, "gemini-3-pro-image-preview" - ) == { + assert map_openai_size_to_gemini_image_config(size, "gemini-3-pro-image-preview") == { "aspectRatio": expected_aspect_ratio, "imageSize": expected_image_size, } @@ -7598,9 +7431,7 @@ def test_gemini_image_generation_openai_size_auto_uses_google_defaults(size: str map_openai_size_to_gemini_image_config, ) - assert map_openai_size_to_gemini_image_config( - size, "gemini-3-pro-image-preview" - ) is None + assert map_openai_size_to_gemini_image_config(size, "gemini-3-pro-image-preview") is None def test_gemini_imagen_models_use_predict_endpoint(): @@ -7610,13 +7441,9 @@ def test_gemini_imagen_models_use_predict_endpoint(): from unittest.mock import patch, MagicMock from litellm.types.utils import ImageResponse, ImageObject - with patch( - "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post" - ) as mock_post: + with patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") as mock_post: mock_http_response = MagicMock() - mock_http_response.json.return_value = { - "predictions": [{"bytesBase64Encoded": "test_base64_image_data"}] - } + mock_http_response.json.return_value = {"predictions": [{"bytesBase64Encoded": "test_base64_image_data"}]} mock_http_response.status_code = 200 mock_post.return_value = mock_http_response @@ -7632,9 +7459,7 @@ def test_gemini_imagen_models_use_predict_endpoint(): mock_post.assert_called_once() call_args = mock_post.call_args - called_url = ( - call_args[0][0] if call_args[0] else call_args.kwargs.get("url", "") - ) + called_url = call_args[0][0] if call_args[0] else call_args.kwargs.get("url", "") assert ":predict" in called_url assert "imagen-3.0-generate-001" in called_url @@ -7797,11 +7622,7 @@ async def test_claude_tool_use_with_gemini(): if "data: " in chunk_str: try: - data_line = [ - line - for line in chunk_str.split("\n") - if line.startswith("data: ") - ][0] + data_line = [line for line in chunk_str.split("\n") if line.startswith("data: ")][0] json_str = data_line[6:] chunk_data = json.loads(json_str) @@ -7814,24 +7635,15 @@ async def test_claude_tool_use_with_gemini(): if ( chunk_data.get("type") == "message_delta" - and chunk_data.get("delta", {}).get("stop_reason") - is not None + and chunk_data.get("delta", {}).get("stop_reason") is not None and "usage" in chunk_data ): has_usage_in_message_delta = True usage = chunk_data["usage"] - assert ( - "input_tokens" in usage - ), "input_tokens should be present in usage" - assert ( - "output_tokens" in usage - ), "output_tokens should be present in usage" - assert isinstance( - usage["input_tokens"], int - ), "input_tokens should be an integer" - assert isinstance( - usage["output_tokens"], int - ), "output_tokens should be an integer" + assert "input_tokens" in usage, "input_tokens should be present in usage" + assert "output_tokens" in usage, "output_tokens should be present in usage" + assert isinstance(usage["input_tokens"], int), "input_tokens should be an integer" + assert isinstance(usage["output_tokens"], int), "output_tokens should be an integer" print(f"Found usage in message_delta: {usage}") except (json.JSONDecodeError, IndexError) as e: @@ -7846,9 +7658,7 @@ async def test_claude_tool_use_with_gemini(): assert is_content_block_tool_use, "content_block_tool_use should be present" assert is_partial_json, "partial_json should be present" - assert ( - has_usage_in_message_delta - ), "Usage should be present in message_delta with stop_reason" + assert has_usage_in_message_delta, "Usage should be present in message_delta with stop_reason" assert is_content_block_stop, "is_content_block_stop should be present" @@ -7877,28 +7687,22 @@ def test_gemini_reasoning_effort_minimal(): ) request_body = raw_request["raw_request_body"] - assert ( - "generationConfig" in request_body - ), f"Model {model} should have generationConfig" + assert "generationConfig" in request_body, f"Model {model} should have generationConfig" generation_config = request_body["generationConfig"] - assert ( - "thinkingConfig" in generation_config - ), f"Model {model} should have thinkingConfig" + assert "thinkingConfig" in generation_config, f"Model {model} should have thinkingConfig" thinking_config = generation_config["thinkingConfig"] - assert ( - "thinkingBudget" in thinking_config - ), f"Model {model} should have thinkingBudget" + assert "thinkingBudget" in thinking_config, f"Model {model} should have thinkingBudget" actual_budget = thinking_config["thinkingBudget"] - assert ( - actual_budget == expected_min_budget - ), f"Model {model} should map 'minimal' to {expected_min_budget} tokens, got {actual_budget}" + assert actual_budget == expected_min_budget, ( + f"Model {model} should map 'minimal' to {expected_min_budget} tokens, got {actual_budget}" + ) - assert thinking_config.get( - "includeThoughts", True - ), f"Model {model} should have includeThoughts=True for minimal reasoning effort" + assert thinking_config.get("includeThoughts", True), ( + f"Model {model} should have includeThoughts=True for minimal reasoning effort" + ) try: raw_request = return_raw_request( @@ -7913,9 +7717,7 @@ def test_gemini_reasoning_effort_minimal(): request_body = raw_request["raw_request_body"] generation_config = request_body["generationConfig"] thinking_config = generation_config["thinkingConfig"] - assert ( - thinking_config["thinkingBudget"] == 128 - ), "Unknown model should use generic fallback of 128 tokens" + assert thinking_config["thinkingBudget"] == 128, "Unknown model should use generic fallback of 128 tokens" except Exception as e: print(f"Note: Unknown model test skipped due to: {e}") pass @@ -7938,9 +7740,7 @@ def test_gemini_exception_message_format(): mock_response.text = "Invalid API key provided" mock_response.headers = {} - mock_exception = httpx.HTTPStatusError( - message="Bad Request", request=Mock(), response=mock_response - ) + mock_exception = httpx.HTTPStatusError(message="Bad Request", request=Mock(), response=mock_response) mock_exception.response = mock_response mock_exception.status_code = 400 @@ -7960,9 +7760,9 @@ def test_gemini_exception_message_format(): f"Expected 'GeminiException' in error message, got: {error_message}. " f"This test should fail before the fix is implemented." ) - assert ( - "VertexAIException" not in error_message - ), f"Should not contain 'VertexAIException' in error message, got: {error_message}" + assert "VertexAIException" not in error_message, ( + f"Should not contain 'VertexAIException' in error message, got: {error_message}" + ) def test_reasoning_effort_none_mapping(): @@ -8016,17 +7816,11 @@ def test_gemini_function_args_preserve_unicode(): parsed_args = json.loads(arguments_str) assert parsed_args["message"] == "やあ", "Japanese characters should be preserved" - assert ( - parsed_args["recipient"] == "たけし" - ), "Japanese characters should be preserved" + assert parsed_args["recipient"] == "たけし", "Japanese characters should be preserved" assert "\\u" not in arguments_str, "Should not contain Unicode escape sequences" - assert ( - "やあ" in arguments_str - ), "Original Japanese characters should be in the string" - assert ( - "たけし" in arguments_str - ), "Original Japanese characters should be in the string" + assert "やあ" in arguments_str, "Original Japanese characters should be in the string" + assert "たけし" in arguments_str, "Original Japanese characters should be in the string" parts_spanish = [ { @@ -8079,12 +7873,8 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults(): model="gemini-3-flash", ) - assert ( - "thinkingLevel" not in result - ), "Should not force thinkingLevel for Gemini 3" - assert ( - "thinkingBudget" not in result - ), "Should NOT have thinkingBudget for Gemini 3" + assert "thinkingLevel" not in result, "Should not force thinkingLevel for Gemini 3" + assert "thinkingBudget" not in result, "Should NOT have thinkingBudget for Gemini 3" assert result["includeThoughts"] is True thinking_param_disabled: AnthropicThinkingParam = { @@ -8098,10 +7888,7 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults(): ) assert result_disabled.get("includeThoughts") is False - assert ( - "thinkingLevel" not in result_disabled - or result_disabled.get("thinkingLevel") is None - ) + assert "thinkingLevel" not in result_disabled or result_disabled.get("thinkingLevel") is None thinking_param_zero: AnthropicThinkingParam = { "type": "enabled", @@ -8114,10 +7901,7 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults(): ) assert result_zero["includeThoughts"] is False - assert ( - "thinkingLevel" not in result_zero - or result_zero.get("thinkingLevel") is None - ) + assert "thinkingLevel" not in result_zero or result_zero.get("thinkingLevel") is None result_gemini3flashpreview = VertexGeminiConfig._map_thinking_param( thinking_param=thinking_param, @@ -8203,9 +7987,7 @@ def test_anthropic_thinking_param_to_gemini_2_thinkingBudget(): ) assert "thinkingBudget" in result_gemini2, "Should have thinkingBudget for Gemini 2" - assert ( - "thinkingLevel" not in result_gemini2 - ), "Should NOT have thinkingLevel for Gemini 2" + assert "thinkingLevel" not in result_gemini2, "Should NOT have thinkingLevel for Gemini 2" assert result_gemini2["includeThoughts"] is True assert result_gemini2["thinkingBudget"] == 10000 @@ -8241,12 +8023,8 @@ def test_anthropic_thinking_param_via_map_openai_params(): assert "thinkingConfig" in result, "Should have thinkingConfig in optional_params" thinking_config = result["thinkingConfig"] - assert ( - "thinkingLevel" not in thinking_config - ), "Should not force thinkingLevel for Gemini 3 by default" - assert ( - "thinkingBudget" not in thinking_config - ), "Should NOT have thinkingBudget for Gemini 3" + assert "thinkingLevel" not in thinking_config, "Should not force thinkingLevel for Gemini 3 by default" + assert "thinkingBudget" not in thinking_config, "Should NOT have thinkingBudget for Gemini 3" assert thinking_config["includeThoughts"] is True optional_params_2 = {} @@ -8259,12 +8037,8 @@ def test_anthropic_thinking_param_via_map_openai_params(): assert "thinkingConfig" in result_2, "Should have thinkingConfig in optional_params" thinking_config_2 = result_2["thinkingConfig"] - assert ( - "thinkingBudget" in thinking_config_2 - ), "Should have thinkingBudget for Gemini 2" - assert ( - "thinkingLevel" not in thinking_config_2 - ), "Should NOT have thinkingLevel for Gemini 2" + assert "thinkingBudget" in thinking_config_2, "Should have thinkingBudget for Gemini 2" + assert "thinkingLevel" not in thinking_config_2, "Should NOT have thinkingLevel for Gemini 2" assert thinking_config_2["includeThoughts"] is True assert thinking_config_2["thinkingBudget"] == 10000 @@ -8284,9 +8058,9 @@ def test_gemini_31_flash_lite_reasoning_effort_minimal(): reasoning_effort="minimal", model="gemini-3.1-flash-lite-preview", ) - assert ( - result["thinkingLevel"] == "minimal" - ), f"Expected thinkingLevel='minimal' for gemini-3.1-flash-lite-preview, got '{result['thinkingLevel']}'" + assert result["thinkingLevel"] == "minimal", ( + f"Expected thinkingLevel='minimal' for gemini-3.1-flash-lite-preview, got '{result['thinkingLevel']}'" + ) assert result["includeThoughts"] is True from litellm.utils import return_raw_request @@ -8302,12 +8076,12 @@ def test_gemini_31_flash_lite_reasoning_effort_minimal(): ) generation_config = raw_request["raw_request_body"]["generationConfig"] thinking_config = generation_config["thinkingConfig"] - assert ( - thinking_config.get("thinkingLevel") == "minimal" - ), f"Expected thinkingLevel='minimal' via full flow, got {thinking_config}" - assert ( - "thinkingBudget" not in thinking_config - ), "gemini-3.1-flash-lite-preview should use thinkingLevel, not thinkingBudget" + assert thinking_config.get("thinkingLevel") == "minimal", ( + f"Expected thinkingLevel='minimal' via full flow, got {thinking_config}" + ) + assert "thinkingBudget" not in thinking_config, ( + "gemini-3.1-flash-lite-preview should use thinkingLevel, not thinkingBudget" + ) @pytest.mark.usefixtures("fake_provider_credentials") @@ -8855,10 +8629,7 @@ async def test_gemini_context_caching_disabled_flag(sync_mode): "content": [ { "type": "text", - "text": "Here is the full text of a complex legal agreement {}".format( - uuid.uuid4() - ) - * 4000, + "text": "Here is the full text of a complex legal agreement {}".format(uuid.uuid4()) * 4000, "cache_control": {"type": "ephemeral"}, } ], @@ -8894,9 +8665,7 @@ async def test_gemini_context_caching_disabled_flag(sync_mode): else: client = AsyncHTTPHandler(concurrent_limit=1) - with patch.object( - client, "post", side_effect=mock_gemini_request - ) as mock_client: + with patch.object(client, "post", side_effect=mock_gemini_request) as mock_client: try: if sync_mode: response = litellm.completion( @@ -8919,26 +8688,22 @@ async def test_gemini_context_caching_disabled_flag(sync_mode): print(e) # When caching is disabled, should only make 1 call (no separate cache creation call) - assert ( - mock_client.call_count == 1 - ), f"Expected 1 call when caching is disabled, got {mock_client.call_count}" + assert mock_client.call_count == 1, ( + f"Expected 1 call when caching is disabled, got {mock_client.call_count}" + ) first_call_args = mock_client.call_args_list[0].kwargs first_call_positional_args = mock_client.call_args_list[0].args print(f"first_call_args with caching disabled: {first_call_args}") - print( - f"first_call_positional_args with caching disabled: {first_call_positional_args}" - ) + print(f"first_call_positional_args with caching disabled: {first_call_positional_args}") # Assert that cachedContents is NOT in the URL when caching is disabled url = first_call_args.get( "url", first_call_positional_args[0] if first_call_positional_args else "", ) - assert ( - "cachedContents" not in url - ), "cachedContents should not be in URL when caching is disabled" + assert "cachedContents" not in url, "cachedContents should not be in URL when caching is disabled" finally: # Restore original flag value @@ -9031,9 +8796,7 @@ def test_gemini_function_call_parameter_in_messages(): "contents": [ { "role": "user", - "parts": [ - {"text": "search for weather in boston (use `search`)"} - ], + "parts": [{"text": "search for weather in boston (use `search`)"}], }, { "role": "model", @@ -9052,17 +8815,13 @@ def test_gemini_function_call_parameter_in_messages(): { "function_response": { "name": "search", - "response": { - "content": "The current weather in Boston is 22°F." - }, + "response": {"content": "The current weather in Boston is 22°F."}, } } ], }, ], - "system_instruction": { - "parts": [{"text": "Use search for most queries."}] - }, + "system_instruction": {"parts": [{"text": "Use search for most queries."}]}, "tools": [ { "function_declarations": [ @@ -9137,9 +8896,7 @@ def test_gemini_function_call_parameter_in_messages_2(): { "function_response": { "name": "search", - "response": { - "content": "The weather in Boston is 100 degrees." - }, + "response": {"content": "The weather in Boston is 100 degrees."}, } } ], @@ -9517,9 +9274,7 @@ def test_vertex_ai_cache_creation_kwarg_cannot_change_usage_or_cost() -> None: mock_response.status_code = 200 mock_response.headers = {"Content-Type": "application/json"} mock_response.json.return_value = { - "candidates": [ - {"content": {"parts": [{"text": "hi"}], "role": "model"}, "finishReason": "STOP", "index": 0} - ], + "candidates": [{"content": {"parts": [{"text": "hi"}], "role": "model"}, "finishReason": "STOP", "index": 0}], "usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 5, "totalTokenCount": 15}, } client = HTTPHandler() @@ -9585,14 +9340,51 @@ def load_vertex_ai_credentials(): # Gemini prebuilt TTS voices per https://ai.google.dev/gemini-api/docs/speech-generation (page dated 2026-10-01, read 2026-10-07) GEMINI_TTS_PREBUILT_VOICES: Final[frozenset[str]] = frozenset( { - "Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus", "Aoede", "Callirrhoe", "Autonoe", - "Enceladus", "Iapetus", "Umbriel", "Algieba", "Despina", "Erinome", "Algenib", "Rasalgethi", - "Laomedeia", "Achernar", "Alnilam", "Schedar", "Gacrux", "Pulcherrima", "Achird", "Zubenelgenubi", - "Vindemiatrix", "Sadachbia", "Sadaltager", "Sulafat", + "Zephyr", + "Puck", + "Charon", + "Kore", + "Fenrir", + "Leda", + "Orus", + "Aoede", + "Callirrhoe", + "Autonoe", + "Enceladus", + "Iapetus", + "Umbriel", + "Algieba", + "Despina", + "Erinome", + "Algenib", + "Rasalgethi", + "Laomedeia", + "Achernar", + "Alnilam", + "Schedar", + "Gacrux", + "Pulcherrima", + "Achird", + "Zubenelgenubi", + "Vindemiatrix", + "Sadachbia", + "Sadaltager", + "Sulafat", } ) OPENAI_TTS_VOICES: Final[tuple[str, ...]] = ( - "alloy", "ash", "ballad", "cedar", "coral", "echo", "fable", "marin", "onyx", "sage", "shimmer", "verse", + "alloy", + "ash", + "ballad", + "cedar", + "coral", + "echo", + "fable", + "marin", + "onyx", + "sage", + "shimmer", + "verse", ) @@ -9678,3 +9470,289 @@ def test_streaming_usage_reads_cache_creation_tokens_recorded_on_logging_obj(): assert chunk.usage.prompt_tokens == 1536 + 1280 assert chunk.usage.cache_creation_input_tokens == 1280 + + +THOUGHT_SUMMARY_REQUESTS: Final = ( + pytest.param("gemini-2.5-flash", {"reasoning_effort": "minimal"}, id="budget-minimal"), + pytest.param("gemini-2.5-flash", {"reasoning_effort": "low"}, id="budget-low"), + pytest.param("gemini-2.5-flash", {"reasoning_effort": "medium"}, id="budget-medium"), + pytest.param("gemini-2.5-flash", {"reasoning_effort": "high"}, id="budget-high"), + pytest.param("gemini-3.6-flash", {"reasoning_effort": "minimal"}, id="level-minimal"), + pytest.param("gemini-3.8-flash", {"reasoning_effort": "minimal"}, id="level-minimal-mapped-to-low"), + pytest.param("gemini-3.8-flash", {"reasoning_effort": "low"}, id="level-low"), + pytest.param("gemini-3.8-flash", {"reasoning_effort": "medium"}, id="level-medium"), + pytest.param("gemini-3.8-flash", {"reasoning_effort": "high"}, id="level-high"), + pytest.param("gemini-2.5-flash", {"thinking": {"type": "enabled", "budget_tokens": 2048}}, id="budget-thinking"), + pytest.param("gemini-3.8-flash", {"thinking": {"type": "enabled", "budget_tokens": 2048}}, id="level-thinking"), +) + + +async def _sent_thinking_config(model: str, thinking_params: dict[str, object]) -> dict[str, object]: + captured: Final[dict[str, dict]] = {} + + def upstream(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "candidates": [{"content": {"parts": [{"text": "hi"}], "role": "model"}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}, + }, + request=request, + ) + + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) + await litellm.acompletion( + model=f"gemini/{model}", + messages=[{"role": "user", "content": "hi"}], + api_key="fake-gemini-key", + client=client, + **thinking_params, + ) + return captured["body"]["generationConfig"]["thinkingConfig"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "thinking_params"), THOUGHT_SUMMARY_REQUESTS) +async def test_gemini_thought_summaries_off_keeps_thinking_config_otherwise_unchanged( + model: str, + thinking_params: dict[str, object], + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + summaries_on: Final = await _sent_thinking_config(model, thinking_params) + monkeypatch.setattr(litellm, "gemini_include_thoughts_default", False) + summaries_off: Final = await _sent_thinking_config(model, thinking_params) + + assert summaries_on["includeThoughts"] is True + assert summaries_off == {**summaries_on, "includeThoughts": False} + + +PER_REQUEST_THOUGHT_SUMMARY_ASKS: Final = ( + pytest.param( + "gemini-3.8-flash", + {"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "summarized"}}, + False, + True, + id="level-display-summarized-beats-off", + ), + pytest.param( + "gemini-3.8-flash", + {"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "omitted"}}, + True, + True, + id="level-display-omitted-leaves-the-default-on", + ), + pytest.param( + "gemini-2.5-flash", + {"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "summarized"}}, + False, + True, + id="budget-display-summarized-beats-off", + ), + pytest.param( + "gemini-2.5-flash", + {"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "omitted"}}, + True, + True, + id="budget-display-omitted-leaves-the-default-on", + ), + pytest.param( + "gemini-3.8-flash", + {"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}, + False, + True, + id="level-display-updates-beats-off", + ), + pytest.param( + "gemini-3.8-flash", + {"reasoning_effort": {"effort": "medium", "summary": "auto"}}, + False, + True, + id="level-effort-summary-beats-off", + ), + pytest.param( + "gemini-2.5-flash", + {"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, + False, + True, + id="budget-effort-summary-beats-off", + ), + pytest.param( + "gemini-3.8-flash", + {"thinking": {"type": "enabled", "budget_tokens": 2048, "summary": "auto"}}, + False, + True, + id="level-thinking-summary-beats-off", + ), + pytest.param( + "gemini-2.5-flash", + {"thinking": {"type": "enabled", "budget_tokens": 2048, "summary": "detailed"}}, + False, + True, + id="budget-thinking-summary-beats-off", + ), + pytest.param( + "gemini-3.8-flash", + {"reasoning_effort": "medium", "reasoning_summary": "auto"}, + False, + True, + id="level-reasoning-summary-alias-beats-off", + ), + pytest.param( + "gemini-2.5-flash", + {"reasoning_effort": "medium", "reasoningSummary": "detailed"}, + False, + True, + id="budget-reasoning-summary-camel-alias-beats-off", + ), + pytest.param( + "gemini-3.8-flash", + {"reasoning_effort": "none", "reasoning_summary": "auto"}, + True, + False, + id="effort-none-ignores-summary-alias", + ), + pytest.param( + "gemini-3.8-flash", + {"reasoning_effort": {"effort": "medium"}}, + False, + False, + id="effort-without-summary-keeps-off", + ), + pytest.param( + "gemini-3.8-flash", + {"reasoning_effort": {"effort": "medium"}, "reasoning_summary": "auto"}, + False, + True, + id="effort-dict-takes-the-summary-alias", + ), + pytest.param( + "gemini-3.8-flash", + {"reasoning_effort": {"effort": "none", "summary": "auto"}}, + True, + False, + id="effort-none-ignores-summary", + ), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "thinking_params", "gateway_default", "expected"), PER_REQUEST_THOUGHT_SUMMARY_ASKS) +async def test_gemini_per_request_thought_summary_ask_beats_gateway_default( + model: str, + thinking_params: dict[str, object], + gateway_default: bool, + expected: bool, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "gemini_include_thoughts_default", gateway_default) + + sent: Final = await _sent_thinking_config(model, thinking_params) + + assert sent["includeThoughts"] is expected + + +def _includes_thoughts_client(sent: list[bool]) -> AsyncHTTPHandler: + def upstream(request: httpx.Request) -> httpx.Response: + sent.append(json.loads(request.content)["generationConfig"]["thinkingConfig"]["includeThoughts"]) + return httpx.Response( + 200, + json={ + "candidates": [{"content": {"parts": [{"text": "hi"}], "role": "model"}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}, + }, + request=request, + ) + + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) + return client + + +async def _chat_summary_ask(client: AsyncHTTPHandler, asks: bool) -> None: + await litellm.acompletion( + model="gemini/gemini-3.8-flash", + messages=[{"role": "user", "content": "hi"}], + api_key="fake-gemini-key", + client=client, + reasoning_effort="medium", + **({"reasoning_summary": "auto"} if asks else {}), + ) + + +async def _messages_summary_ask(client: AsyncHTTPHandler, asks: bool) -> None: + await litellm.anthropic.messages.acreate( + model="gemini/gemini-3.8-flash", + messages=[{"role": "user", "content": "hi"}], + max_tokens=64, + api_key="fake-gemini-key", + client=client, + thinking={"type": "enabled", "budget_tokens": 2048, **({"display": "summarized"} if asks else {})}, + ) + + +async def _responses_summary_ask(client: AsyncHTTPHandler, asks: bool) -> None: + await litellm.aresponses( + model="gemini/gemini-3.8-flash", + input="hi", + api_key="fake-gemini-key", + client=client, + reasoning={"effort": "medium", **({"summary": "auto"} if asks else {})}, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "send", + (_chat_summary_ask, _messages_summary_ask, _responses_summary_ask), + ids=("chat-reasoning-summary-alias", "messages-display-summarized", "responses-reasoning-summary"), +) +async def test_gemini_summary_ask_turns_thoughts_on_and_logs_the_same_reasoning_effort( + send: Callable[[AsyncHTTPHandler, bool], Awaitable[None]], + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + logged: Final[list[object]] = [] + + class ReasoningEffortLogger(CustomLogger): + async def async_log_success_event( + self, + kwargs: Mapping[str, Mapping[str, Mapping[str, object]]], + response_obj: object, + start_time: object, + end_time: object, + ) -> None: + logged.append(kwargs["standard_logging_object"]["model_parameters"].get("reasoning_effort")) + + monkeypatch.setattr(litellm, "gemini_include_thoughts_default", False) + monkeypatch.setattr(litellm, "callbacks", [ReasoningEffortLogger()]) + sent: Final[list[bool]] = [] + client: Final = _includes_thoughts_client(sent) + + await send(client, False) + await send(client, True) + await asyncio.sleep(0) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + assert sent == [False, True] + assert len(logged) == 2 + assert isinstance(logged[0], str) + assert logged[1] == logged[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("gateway_default", "expected"), + (("false", False), (" Off ", False), ("0", False), ("true", True), (None, True), ("not-a-flag", True)), +) +async def test_gemini_sends_a_boolean_for_a_gateway_default_written_as_text_or_null( + gateway_default: object, expected: bool, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "gemini_include_thoughts_default", gateway_default) + + sent: Final = await _sent_thinking_config("gemini-3.8-flash", {"reasoning_effort": "high"}) + + assert sent["includeThoughts"] is expected diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 0d55fc35e1d..19bbb6481fb 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -3109,7 +3109,6 @@ class TestToolTransformation: ("bedrock/converse/global.anthropic.claude-sonnet-5", "bedrock_converse"), ("anthropic.claude-sonnet-4-5-20250929-v1:0", "bedrock"), ("claude-sonnet-5", "vertex_ai"), - ("gemini-3.1-pro-preview", "vertex_ai"), ("moonshotai.kimi-k2-thinking", "bedrock_mantle"), ], ) @@ -3117,8 +3116,8 @@ class TestToolTransformation: """ A Responses request carrying ``reasoning.summary`` must still reach a chat provider as a plain ``reasoning_effort`` string. ``summary`` is Responses-only, and forwarding the whole - object turns reasoning off: Bedrock Converse and Vertex silently discard a non-string - ``reasoning_effort``, and Bedrock Mantle rejects the request outright. + object turns reasoning off: Bedrock Converse and Claude on Vertex silently discard a + non-string ``reasoning_effort``, and Bedrock Mantle rejects the request outright. """ responses_api_request = {"reasoning": {"effort": "medium", "summary": "auto"}} @@ -3131,6 +3130,42 @@ class TestToolTransformation: assert result["reasoning_effort"] == "medium" + @pytest.mark.parametrize( + "model, custom_llm_provider", + [("gemini-3.1-pro-preview", "vertex_ai"), ("gemini-2.5-flash", "gemini")], + ) + @pytest.mark.parametrize( + "reasoning, sends_thoughts", + [({"effort": "medium", "summary": "auto"}, True), ({"effort": "medium"}, False)], + ) + def test_a_summary_ask_turns_gemini_thoughts_on_over_the_gateway_default( + self, + monkeypatch: pytest.MonkeyPatch, + model: str, + custom_llm_provider: str, + reasoning: dict[str, str], + sends_thoughts: bool, + ) -> None: + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig + from litellm.utils import reasoning_effort_with_summary_alias + + monkeypatch.setattr(litellm, "gemini_include_thoughts_default", False) + bridged: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input="hi", + responses_api_request={"reasoning": reasoning}, + custom_llm_provider=custom_llm_provider, + ) + + gemini: Final = VertexGeminiConfig() + reasoning_effort: Final = reasoning_effort_with_summary_alias( + bridged["reasoning_effort"], bridged, accepts_summary=gemini.accepts_reasoning_summary_in_reasoning_effort + ) + mapped: Final = gemini.map_openai_params({"reasoning_effort": reasoning_effort}, {}, model, True) + + assert mapped["thinkingConfig"]["includeThoughts"] is sends_thoughts + assert bridged["reasoning_effort"] == "medium" + @pytest.mark.parametrize( "model, custom_llm_provider", [