mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-17 04:34:25 +00:00
feat(gemini): allow disabling thought summaries while keeping thinking on (#36460)
* feat(gemini): allow disabling thought summaries while keeping thinking on Both reasoning-effort mappers hardcoded includeThoughts to True whenever thinking was on, so thinking ON with summaries OFF was unreachable even though that is the google SDK's own default. Summaries are the narration rather than the reasoning, and for callers that discard them they are pure added latency. Adds litellm.gemini_include_thoughts_default, defaulting to True so nothing changes for anyone who does not set it. It gates the includeThoughts=True branches in the thinkingBudget mapper, the thinkingLevel mapper, and the anthropic-style thinking param mapper. The disable and none paths still force includeThoughts=False: with thinking off there is nothing to summarise. * test(gemini): assert thought-summary flag through the outbound Gemini request * feat(gemini): let a request's thought summary ask override the gateway default thinking.display (summarized, updates, omitted) and a reasoning_effort object carrying summary now decide includeThoughts for that request; gemini_include_thoughts_default applies only when the request says nothing * feat(gemini): let a thought summary ask on /v1/messages and /v1/responses override the gateway default * fix(gemini): read a text gateway default as a boolean and leave display omitted to the flag Normalize gemini_include_thoughts_default through normalize_drop_params so a YAML or env value written as "false" or "0" turns thought text off and a null or unknown value keeps the default on. thinking.display "omitted" no longer counts as a per-request ask, so the gateway flag decides, matching /v1/messages. Adds proxy integration cells for chat, /v1/messages and /v1/responses that prove the flag, every per-request ask, streaming, and a burst across workers. * test(gemini): match the streamed Responses usage, which omits null token details * feat(gemini): read the AI SDK reasoning summary alias and thinking.summary as chat asks * fix(gemini): keep the logged reasoning_effort unchanged and key the response cache on the summary alias A dict reasoning_effort next to the AI SDK reasoning_summary alias now counts as a summary ask, and the alias is part of the cache key so a plain request and an alias ask never share a reply * fix(caching): key the response cache on the summary alias wherever the request carries it The cache key now reads reasoning_summary and reasoningSummary through the same reader as the summary fold, so an alias inside extra_body gets its own key too * test: type the new summary-ask unit tests and mark their locals Final * test: annotate the remaining summary-ask unit tests the PR touches --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Aryan Pardeshi
GitHub
parent
84eeebd509
commit
a844dadcd2
20 changed files with 2841 additions and 674 deletions
|
|
@ -393,6 +393,7 @@ key_alias_pattern: str | None = None
|
|||
enable_gemini_default_thinking_level_low: bool = (
|
||||
False # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping
|
||||
)
|
||||
gemini_include_thoughts_default: bool = True
|
||||
####################
|
||||
logging: bool = True
|
||||
enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ from litellm._internal_context import current_service_target, service_target
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
|
||||
from litellm.integrations.otel.runtime import phase_span
|
||||
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
|
||||
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
|
||||
from litellm.types.caching import *
|
||||
from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
|
||||
|
||||
|
|
@ -376,7 +376,7 @@ class Cache:
|
|||
)
|
||||
return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
|
||||
|
||||
def get_cache_key(self, **kwargs) -> str:
|
||||
def get_cache_key(self, **kwargs: object) -> str:
|
||||
"""
|
||||
Get the cache key for the given arguments.
|
||||
|
||||
|
|
@ -410,13 +410,16 @@ class Cache:
|
|||
if litellm.enable_caching_on_provider_specific_optional_params is True: # feature flagged for now
|
||||
if kwargs[param] is None:
|
||||
continue # ignore None params
|
||||
param_value = kwargs[param]
|
||||
cache_key += f"{param}: {param_value}"
|
||||
cache_key += f"{param}: {kwargs[param]}"
|
||||
|
||||
if is_semantic_cache:
|
||||
cache_key += self._get_semantic_cache_tenant_scope(kwargs)
|
||||
|
||||
hashed_cache_key = Cache._get_hashed_cache_key(cache_key)
|
||||
reasoning_summary: Final = peek_reasoning_summary_aliases(kwargs)
|
||||
reasoning_summary_material: Final = (
|
||||
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
|
||||
)
|
||||
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
|
||||
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
|
||||
verbose_logger.debug(
|
||||
"\nCreated cache key: %s (source material length: %d)",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Mapping
|
||||
from functools import lru_cache
|
||||
from typing import ClassVar, Final
|
||||
|
||||
|
|
@ -22,6 +23,24 @@ from litellm.types.llms.anthropic import AnthropicMessagesRequest
|
|||
from litellm.types.rerank import RerankRequest
|
||||
|
||||
|
||||
def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> object | None:
|
||||
"""Read AI-SDK-style reasoning summary from optional_params or nested extra_body.
|
||||
|
||||
Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped.
|
||||
"""
|
||||
if "reasoningSummary" in optional_params:
|
||||
return optional_params["reasoningSummary"]
|
||||
if "reasoning_summary" in optional_params:
|
||||
return optional_params["reasoning_summary"]
|
||||
extra_body: Final = optional_params.get("extra_body")
|
||||
if isinstance(extra_body, dict):
|
||||
if "reasoningSummary" in extra_body:
|
||||
return extra_body["reasoningSummary"]
|
||||
if "reasoning_summary" in extra_body:
|
||||
return extra_body["reasoning_summary"]
|
||||
return None
|
||||
|
||||
|
||||
class ModelParamHelper:
|
||||
# Cached at class level — deterministic set built from static OpenAI type annotations
|
||||
relevant_logging_args: ClassVar[frozenset[str]] = frozenset()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
|
|
@ -62,6 +63,12 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object):
|
|||
custom_llm_provider: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ChatTargetReasoning:
|
||||
effort: str
|
||||
summary: str | None = None
|
||||
|
||||
|
||||
def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool:
|
||||
"""Return True when any message carries a ``compaction`` content block."""
|
||||
for msg in messages:
|
||||
|
|
@ -441,15 +448,33 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
completion_kwargs["reasoning_effort"] = updated_reasoning_effort
|
||||
|
||||
@staticmethod
|
||||
def _plain_effort_for_chat_target(
|
||||
def _requested_reasoning_summary(thinking: Mapping[str, object]) -> str | None:
|
||||
if thinking.get("type") == "disabled":
|
||||
return None
|
||||
summary: Final = thinking.get("summary")
|
||||
if isinstance(summary, str):
|
||||
return summary
|
||||
display: Final = thinking.get("display")
|
||||
return "auto" if display is not None and display != "omitted" else None
|
||||
|
||||
@staticmethod
|
||||
def _chat_target_reasoning(
|
||||
completion_kwargs: _CompletionKwargs,
|
||||
*,
|
||||
thinking: Mapping[str, object] | None,
|
||||
) -> str | None:
|
||||
reasoning_effort: Final = completion_kwargs.get("reasoning_effort")
|
||||
if not thinking or not isinstance(reasoning_effort, dict) or "summary" not in reasoning_effort:
|
||||
) -> _ChatTargetReasoning | None:
|
||||
if not thinking:
|
||||
return None
|
||||
effort: Final = reasoning_effort.get("effort")
|
||||
reasoning_effort: Final = completion_kwargs.get("reasoning_effort")
|
||||
wraps_summary: Final = isinstance(reasoning_effort, dict) and "summary" in reasoning_effort
|
||||
requested_summary: Final = LiteLLMMessagesToCompletionTransformationHandler._requested_reasoning_summary(
|
||||
thinking
|
||||
)
|
||||
if not wraps_summary and requested_summary is None:
|
||||
return None
|
||||
effort: Final[object] = (
|
||||
reasoning_effort.get("effort") if isinstance(reasoning_effort, dict) else reasoning_effort
|
||||
)
|
||||
model: Final = completion_kwargs.get("model")
|
||||
if not isinstance(effort, str) or not isinstance(model, str) or not model:
|
||||
return None
|
||||
|
|
@ -467,7 +492,12 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
return None
|
||||
if resolved_provider == "litellm_proxy":
|
||||
return None
|
||||
from litellm.responses.bridge_check import responses_api_bridge_check
|
||||
from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary, responses_api_bridge_check
|
||||
|
||||
if requested_summary is not None and chat_target_accepts_reasoning_summary(local_model, resolved_provider):
|
||||
return _ChatTargetReasoning(effort=effort, summary=requested_summary)
|
||||
if not wraps_summary or not isinstance(reasoning_effort, dict):
|
||||
return None
|
||||
|
||||
web_search_options: Final = completion_kwargs.get("web_search_options")
|
||||
tools: Final = completion_kwargs.get("tools")
|
||||
|
|
@ -481,7 +511,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
reasoning_effort=reasoning_effort,
|
||||
api_base=resolved_api_base,
|
||||
)
|
||||
return None if model_info.get("mode") == "responses" else effort
|
||||
return None if model_info.get("mode") == "responses" else _ChatTargetReasoning(effort=effort)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_reasoning_effort(
|
||||
|
|
@ -647,12 +677,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
thinking=thinking,
|
||||
)
|
||||
|
||||
plain_effort: Final = LiteLLMMessagesToCompletionTransformationHandler._plain_effort_for_chat_target(
|
||||
chat_reasoning: Final = LiteLLMMessagesToCompletionTransformationHandler._chat_target_reasoning(
|
||||
completion_kwargs,
|
||||
thinking=thinking,
|
||||
)
|
||||
if plain_effort is not None:
|
||||
completion_kwargs["reasoning_effort"] = plain_effort
|
||||
if chat_reasoning is not None:
|
||||
completion_kwargs["reasoning_effort"] = chat_reasoning.effort
|
||||
if chat_reasoning is not None and chat_reasoning.summary is not None:
|
||||
completion_kwargs["reasoning_summary"] = chat_reasoning.summary
|
||||
|
||||
return completion_kwargs, tool_name_mapping
|
||||
|
||||
|
|
|
|||
|
|
@ -488,6 +488,10 @@ class BaseConfig(ABC):
|
|||
"""
|
||||
return True
|
||||
|
||||
@property
|
||||
def accepts_reasoning_summary_in_reasoning_effort(self) -> bool:
|
||||
return False
|
||||
|
||||
def post_stream_processing(self, stream: object) -> object:
|
||||
"""Hook for providers to post-process streaming responses. Default: pass-through."""
|
||||
return stream
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from litellm.constants import (
|
|||
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO,
|
||||
)
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
encode_tool_call_id_with_signature,
|
||||
|
|
@ -926,11 +927,38 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
# - Requires propertyOrdering
|
||||
optional_params["response_schema"] = self._map_response_schema(value=schema)
|
||||
|
||||
@property
|
||||
def accepts_reasoning_summary_in_reasoning_effort(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _include_thoughts(requested: bool | None) -> bool:
|
||||
if requested is not None:
|
||||
return requested
|
||||
return normalize_drop_params(litellm.gemini_include_thoughts_default) is not False
|
||||
|
||||
@staticmethod
|
||||
def _include_thoughts_requested_by_reasoning_effort(reasoning_effort: object) -> bool | None:
|
||||
match reasoning_effort:
|
||||
case {"summary": str()}:
|
||||
return True
|
||||
case _:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _map_reasoning_effort(effort: str, model: str, reasoning_effort: object) -> GeminiThinkingConfig:
|
||||
include_thoughts: Final = VertexGeminiConfig._include_thoughts_requested_by_reasoning_effort(reasoning_effort)
|
||||
if VertexGeminiConfig._is_gemini_3_or_newer(model):
|
||||
return VertexGeminiConfig._map_reasoning_effort_to_thinking_level(effort, model, include_thoughts)
|
||||
return VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(effort, model, include_thoughts)
|
||||
|
||||
@staticmethod
|
||||
def _map_reasoning_effort_to_thinking_budget(
|
||||
reasoning_effort: str,
|
||||
model: str | None = None,
|
||||
include_thoughts: bool | None = None,
|
||||
) -> GeminiThinkingConfig:
|
||||
sends_thoughts: Final = VertexGeminiConfig._include_thoughts(include_thoughts)
|
||||
if reasoning_effort == "minimal":
|
||||
# Use model-specific minimum thinking budget or fallback
|
||||
# Check for exact matches first, then partial matches
|
||||
|
|
@ -945,22 +973,22 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
|
||||
return {
|
||||
"thinkingBudget": budget,
|
||||
"includeThoughts": True,
|
||||
"includeThoughts": sends_thoughts,
|
||||
}
|
||||
elif reasoning_effort == "low":
|
||||
return {
|
||||
"thinkingBudget": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
|
||||
"includeThoughts": True,
|
||||
"includeThoughts": sends_thoughts,
|
||||
}
|
||||
elif reasoning_effort == "medium":
|
||||
return {
|
||||
"thinkingBudget": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
|
||||
"includeThoughts": True,
|
||||
"includeThoughts": sends_thoughts,
|
||||
}
|
||||
elif reasoning_effort == "high":
|
||||
return {
|
||||
"thinkingBudget": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
|
||||
"includeThoughts": True,
|
||||
"includeThoughts": sends_thoughts,
|
||||
}
|
||||
elif reasoning_effort == "disable":
|
||||
return {
|
||||
|
|
@ -987,6 +1015,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
def _map_reasoning_effort_to_thinking_level(
|
||||
reasoning_effort: str,
|
||||
model: str | None = None,
|
||||
include_thoughts: bool | None = None,
|
||||
) -> GeminiThinkingConfig:
|
||||
"""
|
||||
Map reasoning_effort to thinking_level for Gemini 3+ models.
|
||||
|
|
@ -998,17 +1027,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
GeminiThinkingConfig with thinkingLevel and includeThoughts
|
||||
"""
|
||||
supports_minimal: Final = bool(model) and VertexGeminiConfig._supports_minimal_thinking_level(model)
|
||||
sends_thoughts: Final = VertexGeminiConfig._include_thoughts(include_thoughts)
|
||||
if reasoning_effort == "minimal":
|
||||
if supports_minimal:
|
||||
return {"thinkingLevel": "minimal", "includeThoughts": True}
|
||||
return {"thinkingLevel": "minimal", "includeThoughts": sends_thoughts}
|
||||
else:
|
||||
return {"thinkingLevel": "low", "includeThoughts": True}
|
||||
return {"thinkingLevel": "low", "includeThoughts": sends_thoughts}
|
||||
elif reasoning_effort == "low":
|
||||
return {"thinkingLevel": "low", "includeThoughts": True}
|
||||
return {"thinkingLevel": "low", "includeThoughts": sends_thoughts}
|
||||
elif reasoning_effort == "medium":
|
||||
return {"thinkingLevel": "medium", "includeThoughts": True}
|
||||
return {"thinkingLevel": "medium", "includeThoughts": sends_thoughts}
|
||||
elif reasoning_effort == "high":
|
||||
return {"thinkingLevel": "high", "includeThoughts": True}
|
||||
return {"thinkingLevel": "high", "includeThoughts": sends_thoughts}
|
||||
elif reasoning_effort in ("disable", "none"):
|
||||
return {
|
||||
"thinkingLevel": "minimal" if supports_minimal else "low",
|
||||
|
|
@ -1061,6 +1091,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
status_code=400,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _thinking_asks_for_thoughts(thinking_param: Mapping[str, object]) -> bool:
|
||||
display: Final = thinking_param.get("display")
|
||||
return isinstance(thinking_param.get("summary"), str) or (display is not None and display != "omitted")
|
||||
|
||||
@staticmethod
|
||||
def _map_thinking_param(
|
||||
thinking_param: AnthropicThinkingParam,
|
||||
|
|
@ -1068,6 +1103,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
) -> GeminiThinkingConfig:
|
||||
thinking_enabled: Final = thinking_param.get("type") == "enabled"
|
||||
thinking_budget: Final = thinking_param.get("budget_tokens")
|
||||
asks_for_thoughts: Final = VertexGeminiConfig._thinking_asks_for_thoughts(thinking_param)
|
||||
sends_thoughts: Final = VertexGeminiConfig._include_thoughts(True if asks_for_thoughts else None)
|
||||
|
||||
params: Final[GeminiThinkingConfig] = {}
|
||||
|
||||
|
|
@ -1077,7 +1114,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
if thinking_budget == 0:
|
||||
params["includeThoughts"] = False
|
||||
else:
|
||||
params["includeThoughts"] = True
|
||||
params["includeThoughts"] = sends_thoughts
|
||||
# Follow provider defaults unless explicitly opted into legacy behavior.
|
||||
if litellm.enable_gemini_default_thinking_level_low is True:
|
||||
params["thinkingLevel"] = (
|
||||
|
|
@ -1089,7 +1126,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
else:
|
||||
# For older Gemini models, use thinkingBudget
|
||||
if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget):
|
||||
params["includeThoughts"] = True
|
||||
params["includeThoughts"] = sends_thoughts
|
||||
if thinking_budget is not None and isinstance(thinking_budget, int):
|
||||
params["thinkingBudget"] = thinking_budget
|
||||
|
||||
|
|
@ -1304,14 +1341,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
param_name="reasoning_effort",
|
||||
param_description="thinking_budget",
|
||||
)
|
||||
if VertexGeminiConfig._is_gemini_3_or_newer(model):
|
||||
optional_params["thinkingConfig"] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
|
||||
effort_value, model
|
||||
)
|
||||
else:
|
||||
optional_params["thinkingConfig"] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(
|
||||
effort_value, model
|
||||
)
|
||||
optional_params["thinkingConfig"] = VertexGeminiConfig._map_reasoning_effort(
|
||||
effort=effort_value, model=model, reasoning_effort=value
|
||||
)
|
||||
elif param == "thinking":
|
||||
# Validate no conflict with thinking_level
|
||||
VertexGeminiConfig._validate_thinking_config_conflicts(
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ from litellm.utils import (
|
|||
get_litellm_params,
|
||||
get_optional_params,
|
||||
peek_reasoning_summary_aliases,
|
||||
reasoning_effort_with_summary_alias,
|
||||
strip_reasoning_summary_aliases_from_optional_params,
|
||||
)
|
||||
|
||||
|
|
@ -5681,7 +5682,15 @@ def completion(
|
|||
"allowed_openai_params": allowed_openai_params,
|
||||
"base_model": base_model,
|
||||
}
|
||||
optional_params = get_optional_params(**optional_param_args, **non_default_params)
|
||||
provider_reasoning_effort: Final = reasoning_effort_with_summary_alias(
|
||||
reasoning_effort,
|
||||
non_default_params,
|
||||
accepts_summary=provider_config is not None
|
||||
and provider_config.accepts_reasoning_summary_in_reasoning_effort,
|
||||
)
|
||||
optional_params = get_optional_params(
|
||||
**{**optional_param_args, "reasoning_effort": provider_reasoning_effort}, **non_default_params
|
||||
)
|
||||
processed_non_default_params: Final = pre_process_non_default_params(
|
||||
model=model,
|
||||
passed_params=optional_param_args,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from urllib.parse import urlsplit
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.azure_ai.common_utils import (
|
||||
azure_ai_supports_native_responses,
|
||||
foundry_chat_rejects_function_tools_while_reasoning,
|
||||
|
|
@ -11,7 +12,8 @@ from litellm.llms.azure_ai.common_utils import (
|
|||
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import OpenAIWebSearchOptions
|
||||
from litellm.utils import get_model_info_helper
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager, get_model_info_helper
|
||||
|
||||
_OPENAI_DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
|
||||
_OPENAI_API_HOST: Final = "api.openai.com"
|
||||
|
|
@ -36,6 +38,18 @@ def _resolve_openai_api_base(api_base: str | None) -> str:
|
|||
)
|
||||
|
||||
|
||||
def chat_target_accepts_reasoning_summary(model: str, custom_llm_provider: str | None) -> bool:
|
||||
try:
|
||||
local_model, provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
chat_config: Final = ProviderConfigManager.get_provider_chat_config(
|
||||
model=local_model, provider=LlmProviders(provider)
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # a capability probe must never fail the request it probes for
|
||||
verbose_logger.debug("chat target reasoning summary check failed: %s", e)
|
||||
return False
|
||||
return chat_config is not None and chat_config.accepts_reasoning_summary_in_reasoning_effort
|
||||
|
||||
|
||||
def responses_api_bridge_check(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
|||
from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
|
||||
from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control
|
||||
from litellm.llms.snowflake.chat.transformation import is_snowflake_claude_model
|
||||
from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary
|
||||
from litellm.responses.litellm_completion_transformation.session_handler import (
|
||||
ResponsesSessionHandler,
|
||||
)
|
||||
|
|
@ -785,8 +786,8 @@ class LiteLLMCompletionResponsesConfig:
|
|||
provider either rejects it or silently drops it, and dropping it turns reasoning off while
|
||||
still billing for the turn. ``summary`` has no chat equivalent, so it rides the
|
||||
``reasoning_summary`` alias, which ``litellm.completion`` reassembles into ``{effort,
|
||||
summary}`` when it bridges the model back onto the Responses API, and is sent to nothing
|
||||
else.
|
||||
summary}`` when it bridges the model back onto the Responses API or targets a chat provider
|
||||
whose config maps a summary ask (Gemini's thought summaries), and is sent to nothing else.
|
||||
"""
|
||||
if not reasoning_param:
|
||||
return ResponsesReasoningChatForm(effort=None, summary=None)
|
||||
|
|
@ -807,7 +808,10 @@ class LiteLLMCompletionResponsesConfig:
|
|||
reasoning_summary=summary,
|
||||
api_base=api_base,
|
||||
)
|
||||
return ResponsesReasoningChatForm(effort=effort, summary=summary if bridges_back else None)
|
||||
forwards_summary: Final = bridges_back or (
|
||||
isinstance(effort, str) and chat_target_accepts_reasoning_summary(model, custom_llm_provider)
|
||||
)
|
||||
return ResponsesReasoningChatForm(effort=effort, summary=summary if forwards_summary else None)
|
||||
|
||||
@staticmethod
|
||||
def transform_responses_api_request_to_chat_completion_request(
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
|
|||
match_capability_generalizations,
|
||||
match_fill_missing_generalizations,
|
||||
)
|
||||
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
|
||||
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
|
||||
from litellm.rust_bridge import tokenizer as tokenizer_dispatch
|
||||
|
|
@ -10446,22 +10447,21 @@ def get_non_default_completion_params(kwargs: Mapping[str, object]) -> dict[str,
|
|||
return filter_out_litellm_params(kwargs, excluding=litellm.OPENAI_CHAT_COMPLETION_PARAMS)
|
||||
|
||||
|
||||
def peek_reasoning_summary_aliases(optional_params: dict) -> object | None:
|
||||
"""Read AI-SDK-style reasoning summary from optional_params or nested extra_body.
|
||||
|
||||
Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped.
|
||||
"""
|
||||
if "reasoningSummary" in optional_params:
|
||||
return optional_params["reasoningSummary"]
|
||||
if "reasoning_summary" in optional_params:
|
||||
return optional_params["reasoning_summary"]
|
||||
extra_body: Final = optional_params.get("extra_body")
|
||||
if isinstance(extra_body, dict):
|
||||
if "reasoningSummary" in extra_body:
|
||||
return extra_body["reasoningSummary"]
|
||||
if "reasoning_summary" in extra_body:
|
||||
return extra_body["reasoning_summary"]
|
||||
return None
|
||||
def reasoning_effort_with_summary_alias(
|
||||
reasoning_effort: object, params: Mapping[str, object], accepts_summary: bool
|
||||
) -> object:
|
||||
summary: Final = peek_reasoning_summary_aliases(params)
|
||||
if not accepts_summary or not isinstance(summary, str):
|
||||
return reasoning_effort
|
||||
match reasoning_effort:
|
||||
case str():
|
||||
return {"effort": reasoning_effort, "summary": summary}
|
||||
case {"summary": _}:
|
||||
return reasoning_effort
|
||||
case {"effort": str() as effort}:
|
||||
return {"effort": effort, "summary": summary}
|
||||
case _:
|
||||
return reasoning_effort
|
||||
|
||||
|
||||
def strip_reasoning_summary_aliases_from_optional_params(
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
63
tests/integration/translation/gemini_thoughts.py
Normal file
63
tests/integration/translation/gemini_thoughts.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import JSON_OBJECT, Gateway, list_value, object_value
|
||||
from integration._support.process import OwnedProxy, owned_proxy_process
|
||||
from integration._support.wire import Request
|
||||
from pydantic import JsonValue
|
||||
|
||||
GEMINI_2_5_FLASH_DEPLOYMENT: Final[Mapping[str, JsonValue]] = {
|
||||
"model_name": "gemini/gemini-2.5-flash",
|
||||
"litellm_params": {
|
||||
"model": "gemini/gemini-2.5-flash",
|
||||
"api_base": "http://127.0.0.1:8191",
|
||||
"api_key": "synthetic-gemini-key",
|
||||
},
|
||||
}
|
||||
LONG_SUMMARY: Final = "detailed " * 600
|
||||
|
||||
|
||||
def with_thinking_config(
|
||||
request: Mapping[str, JsonValue], thinking_config: Mapping[str, JsonValue]
|
||||
) -> Mapping[str, JsonValue]:
|
||||
return {
|
||||
**request,
|
||||
"generationConfig": {**object_value(request["generationConfig"]), "thinkingConfig": dict(thinking_config)},
|
||||
}
|
||||
|
||||
|
||||
def sent_thinking_config(request: Request) -> Mapping[str, JsonValue]:
|
||||
body: Final = JSON_OBJECT.validate_json(request.body)
|
||||
return object_value(object_value(body["generationConfig"])["thinkingConfig"])
|
||||
|
||||
|
||||
@contextmanager
|
||||
def gemini_thoughts_proxy(
|
||||
gateway: Gateway, directory: Path, default: JsonValue, *, workers: int = 1
|
||||
) -> Iterator[OwnedProxy]:
|
||||
stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
config: Final = directory / "gemini-thoughts.yaml"
|
||||
config.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
**stock,
|
||||
"model_list": [*list_value(stock["model_list"]), dict(GEMINI_2_5_FLASH_DEPLOYMENT)],
|
||||
"litellm_settings": {
|
||||
**object_value(stock["litellm_settings"]),
|
||||
"gemini_include_thoughts_default": default,
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
with owned_proxy_process(gateway, directory, {}, config=config, workers=workers) as owned:
|
||||
yield owned
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def thoughts_off(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
with gemini_thoughts_proxy(session_gateway, tmp_path_factory.mktemp("gemini-thoughts-off"), False) as owned:
|
||||
yield owned.gateway
|
||||
|
|
@ -0,0 +1,373 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, object_value
|
||||
from integration._support.process import graceful_stop_seconds
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration._support.wire import Reply
|
||||
from integration.translation.case import SSEFrame, TranslationTestCase
|
||||
from integration.translation.gemini_thoughts import LONG_SUMMARY, thoughts_off, with_thinking_config
|
||||
from integration.translation.messages.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE, GEMINI_MESSAGES_RESPONSE
|
||||
from integration.translation.runner import (
|
||||
assert_provider_request,
|
||||
assert_translation,
|
||||
encode_sse_frame,
|
||||
parse_sse_frames,
|
||||
)
|
||||
from pydantic import JsonValue
|
||||
|
||||
__all__ = ("thoughts_off",)
|
||||
|
||||
pytestmark = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE: Final = (
|
||||
"EscLCsQLAWkUfRP4jtXWjNAugx5FeAwxasL/kKzH9gXUHHB2cTK13h4C8C6tKVYMKP5rXoPq4yhoXSrsDiIjUEpuAjdQYYCz"
|
||||
"Wvx21tGAN4iBA4VELBqPOi92vo9YVGpczJ/XVREq9KNRFIkyMzWyEzR1lBbJFA4detKHzSVJ7VQ1iRBn9jk6xvknTCxQhbnl"
|
||||
"Pct8TfMo4XluFsM2PY3qAqoX1DjkrjQNe4nrN9Yoglq8Bwr2ayyc3DUNkNJDu0DQN20W6HU7fbeqSizEOsvGIvJVEUZhMOFF"
|
||||
"0r8NyzUGkNiiAZ+TEupIV+G0eTr6sVlMPLS1N74ryQ55sevdj6irenHlP19IwkiDNhy6ApRIoblY5I1uJVt4lj4tQyACGMAE"
|
||||
"VOTg3Gb78hPgJ+8cF3ukmRF0H1bpYH3n/mpnkqiggfK3vm7fVnsCy03W7Keyl7xd7Gfto/HL7vPCta/sO6ZmZm28JvdCr/Ce"
|
||||
"X/W82nHuEqcjiR8fNPk413rL/geE4Lppru1H9ESWBICXzwl6pLxaBBNr8BD/jGMvlYA/ZEcBfKWtYyzow3+yp+4M0fsY1Ah7"
|
||||
"pxVCcCbivMRv2X5J5m9btU1IIm2REJicjMSWCxh4lQMEuuk2FaPzge+wXROC/Zx0HwNw2iPw6VpziTOByACQxXOE7RdWULq1"
|
||||
"3rkbyXCMkuAfG281SiylOnJ3I3tkbRcFoTaY5DcMBKYCCmoJJZPO1HKJ2DMw5Spz1HDftQ7lg9df5FDoZbmehUZKfY+j/2LN"
|
||||
"//Wr5OEUmuBpojC5ht3YYgXaxclo/pa70KnLHlQWHPZ3nqdB2ru0tqgOaXSv9KZNhtvXgagLsmffQ/rm5QiRoEeG4oZ5E6Ga"
|
||||
"VLQB9OvWCDNdzgNVzss0tveNcG/cttJBnkSd5/azHSvzkCDQYh4J/GT3zDVvOyOayWTqzGSLgIoFGpTES7Vsbpph62apefvO"
|
||||
"+ZAlxPNNo/3tHncHzGc8Pku07fF1yhiKdGRV1OjYndfQv6UuDc3G7n2oU90iCKWRoCqOYAotzw1yCh4lbsSpEWO2iB9kWW93"
|
||||
"mJsdqQBdk6e9UiuHdJIOfQ+cOVNOrltXXtZZ+T2mKFTFaWaN1F7MMH+xIX10dcnsWzq5o0Qz/tpwEfFOaQ5j43ez5BJdE5yD"
|
||||
"wj4sDIdY57jAQ4141jU2oHpJCjY5Obhd4692k/FRtN7IMWMn5rQS+uTctWOh/27qatdy+QeAMPZseT4Yoi3W6W//Um9+sJY4"
|
||||
"RjJ0Q27Fjb/wEqBkGsqx1sIidQc19sZWa2jQxVgG3T8xHYCMbgkqT/OHJaHGa4gxo78ajhdQnGYNWtvuRlGDpdUsxcNfF+6z"
|
||||
"kbqWgcxKleRnQ1jKKZKS9GeiLoJnBRk8wIgmnrleblMB0GRj8i124HKLTfAm3WHlqy3xpACI3AIxPbk1GrATdkrb496eecVG"
|
||||
"ko9YTrsdgOn5RV6Ggy/Rxr5wAAvPKBrqx/u1Whyr05pge9AZcGAtFRIZMWRCBnNg+oHE8M1EVipx3wQQXFDFO27Jpr3IandO"
|
||||
"GgpbjbpwcW0fnDKLw7Ndt8SCyyVUEZwZ4Vp4tXJCOv9rzIMHJi+68Upwp5ILirrBFT5vKs//ub6l5hHzEfZty66+DvXyrpqT"
|
||||
"VLYRnYMEO10Te24QyYVCO4qSnbJzfwTv5Kp4ODP2ONdSuJgkyYy78MzZ926H1/8rOlTEbWpAYb+QJwrt1nBJBkWRNDi2QNrl"
|
||||
"UlFKTXZNLNXTP/rHHqfw1us8bhwe4Up2BbECPO+8F0tgSifXusnShtsRwpW7o+z/iXpnwuWePH92YiORo43mdoG4MJ54Emwf"
|
||||
"B7x9ftguHpq1ZDOBoa3frE8l8a5RIEHemkzQsLlPyZSB+eQgnXrToXwUeEfR3EHG0vSCfokMLk0HinhWGa6hgIyJZQAdUIg4"
|
||||
"GALvjSsrGEHwJaeLNNn/9EscJOEGyFQFMEy+sVIkL6Pj2+iZuG0OKahk"
|
||||
)
|
||||
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE: Final[Mapping[str, JsonValue]] = {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"text": "I need to inspect `shop/validation.py` next to examine how `validate_quantity` handles zero, negative, and non-integer quantities.",
|
||||
},
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "read_file",
|
||||
"args": {
|
||||
"end_line": 40,
|
||||
"path": "shop/validation.py",
|
||||
"start_line": 1,
|
||||
},
|
||||
"id": "call_412671",
|
||||
},
|
||||
"thoughtSignature": GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE,
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
"finishMessage": "Model generated function call(s).",
|
||||
},
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 354,
|
||||
"candidatesTokenCount": 64,
|
||||
"totalTokenCount": 739,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": "TEXT",
|
||||
"tokenCount": 354,
|
||||
},
|
||||
],
|
||||
"thoughtsTokenCount": 321,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "1C3RatW-Jtuc_uMPhaLroA0",
|
||||
}
|
||||
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_THOUGHT_SIGNATURE: Final = (
|
||||
"Eo0LCooLAWkUfRMuNip0QePQVa4NCyMrc0dwJR2qI/PSRcyScXfwpZhjBJVTorCs5ei6D4xzVf9attfTHx8bOBCX9wXynQ6V"
|
||||
"JxU6uN562z5VAyrgoGv2nVGJk/mT7SnzIFiOc4RN+xebNGO0BIDf95xyOMeYP+ZHXquqtx1SnQXw7V6p81wTA4cGMnqjNE7F"
|
||||
"Vs7OqlcpvzeFUmE8gl/Cy1a+u5SgLHwfnvOR3uzIX3u8kdk//dzvCKqXYaRStDf0dsrJICex/9WoOBfZKCujLBMK7CCjKEWF"
|
||||
"nqERtFxXayTUKz1dvvBkcZ+MbhtrG3RFkBegqokQn9lus3icwBAfkZ9b3ps42j/mYw7DfjR+3Dg/Pyz8mgJVRC312QRxt+70"
|
||||
"yOCU8POm/GTHJAZvvxdm9U4eiYBs3789F5oV34YzZl1+pcE4WVRwiFkB0vi17xy5XJFYTtzuk9d9ySr5Fz1de6nTKWk6TpuX"
|
||||
"PM4q8bhX51kJWPCr1W4oVn1GhKpCtRyrXzKhuMsWcHXu/yKHUVjxyEFY+cTsVtHfrMHsIhMHyNnBqz2JclGXtFZScaq6AZ8g"
|
||||
"0JZPqqAXgfgNyx2RaxoaCtITznOlBwMC1hTAZCrtjhHQNvGsAEdaRBAnZRFgeDHkuL7r7D9tr38UdxaUf/R5vfl5eO0ksj4X"
|
||||
"bdeUG/LAnWP2CLUVjIoF50o4ZHKRy5UT/Mhr45986ePmtWnIN9t6dXOCB5I4qoEaljwkDn+llwZMbPq1orHdSrUAR6flcl3h"
|
||||
"qVutnVOFgN3NeHvNkrkjUCvckiQL/ZBiG5YeDKfMP+XECDWIoTS2/DCJ7SX6d44B2yJ/PO2PH3nm7hr81U7r7xYKly4/2YFX"
|
||||
"8bAsFJt4lGihfDHXFOr/VbQQd/CobZtLV9UZP7m1wUGXfL0knP31bdFWDKmSp/2giddbpIsGQf6j7h+YiE3M6QEfvRi7kaDX"
|
||||
"CQ16O/4XJzG85/BxPw8ssR1cN+xLM95pZkDQO2MN1rxLVogF+T+Xc0M2SSjwbahM4nwYmisOqXv4Wuy0V7hQYJUySCvHabdH"
|
||||
"nmTi+3mIfs9/zzHT3QUGZvAumBCZYfz1kqyz6mytXSwQbhhktmDLgIaj1abGm2gSn10fxl8pDm/mTq8/owS9aNPyDYChNBMr"
|
||||
"NGYEBMXA4DtZy9TjybzMXKGPi4d/M7EZM5sC33WnCl2JovveyeZUNPWkZTV0+TjSSqk8C488huCOzv+PgJMvtBq6XiQW9ZV1"
|
||||
"HQWk8XRMK7VMMhuv9sRqy1Lh/h8kIlSWtaCCNwpqIgUlZ+xybbY9dRGdsWq5FoxCU/OLmlSv7BcqXAamxxfIK9inXRKd/xkI"
|
||||
"bR7OCeIaFq97Z5M08LfUpu9WjcXGgvtsppOUB4Rik2uCBVSzA/OqQdY1/N3q+BPYCvhOVVEBOJEEAvH9iAQOtCw4peaCtSLL"
|
||||
"7vLh7s/lIZGBYj5I4zjoOfoSuT9g1x9uvv9h2sJUeP1HsN9Mp95weKBbF3HUN5F/lVRTan2I+Wb2+WnMMbashS1hoVIfk/Bc"
|
||||
"426M3xPy7JDsczXLUP0PAyhAc2+xiFnkI120l4IHT3JK53AArx7svpTxzLOvPffVIcodkyWXHeJeC6eQbD6Elt/wWbx5Ig1L"
|
||||
"mJQP7egSoyqmzu0zpuIC0xVsmDdgnumSi7acTjp1mIsLHHLLaWPcpxaNeiwBQf/7FfKsGqJ6p+I+lzt/rdRDOR4apkwKRwnr"
|
||||
"d5beSf38TJd7L8Zj1qXp36Q2rnvtKubSm7f451w9ZbKsqbMDrwoagNWsfG9kc0CR0wtraSRzBHD6GUfoapPbplOLsSyjzEvX"
|
||||
"i48aG814wRmQk932cd8ETx1a9BIbN/bsOr6stttugYakpvD3b8ckhfXW7eXfpKywBBzu1s1KLrU="
|
||||
)
|
||||
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_FRAMES: Final = (
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"text": "I need to read `shop/validation.py` next to see how `validate_quantity` currently handles zero, negative, and",
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"index": 0,
|
||||
},
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 354,
|
||||
"candidatesTokenCount": 27,
|
||||
"totalTokenCount": 676,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": "TEXT",
|
||||
"tokenCount": 354,
|
||||
},
|
||||
],
|
||||
"thoughtsTokenCount": 295,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "2C3RavD_A77W-sAP9brxyQk",
|
||||
}
|
||||
),
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"text": " non-integer quantities.",
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"index": 0,
|
||||
},
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 354,
|
||||
"candidatesTokenCount": 32,
|
||||
"totalTokenCount": 681,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": "TEXT",
|
||||
"tokenCount": 354,
|
||||
},
|
||||
],
|
||||
"thoughtsTokenCount": 295,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "2C3RavD_A77W-sAP9brxyQk",
|
||||
}
|
||||
),
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "read_file",
|
||||
"args": {
|
||||
"start_line": 1,
|
||||
"end_line": 50,
|
||||
"path": "shop/validation.py",
|
||||
},
|
||||
"id": "call_654547",
|
||||
},
|
||||
"thoughtSignature": GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_THOUGHT_SIGNATURE,
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"index": 0,
|
||||
},
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 354,
|
||||
"candidatesTokenCount": 65,
|
||||
"totalTokenCount": 714,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": "TEXT",
|
||||
"tokenCount": 354,
|
||||
},
|
||||
],
|
||||
"thoughtsTokenCount": 295,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "2C3RavD_A77W-sAP9brxyQk",
|
||||
}
|
||||
),
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"text": "",
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
},
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 354,
|
||||
"candidatesTokenCount": 65,
|
||||
"totalTokenCount": 714,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": "TEXT",
|
||||
"tokenCount": 354,
|
||||
},
|
||||
],
|
||||
"thoughtsTokenCount": 295,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "2C3RavD_A77W-sAP9brxyQk",
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
ADAPTIVE_THINKING: Final[Mapping[str, JsonValue]] = {"type": "adaptive"}
|
||||
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE: Final = replace(
|
||||
GEMINI_3_8_FLASH_TEST_CASE,
|
||||
scenario="thoughts_off",
|
||||
litellm_request={**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "thinking": dict(ADAPTIVE_THINKING)},
|
||||
expected_provider_request=with_thinking_config(
|
||||
GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, {"thinkingLevel": "high", "includeThoughts": False}
|
||||
),
|
||||
mock_provider_response=GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE,
|
||||
expected_litellm_response={
|
||||
**GEMINI_MESSAGES_RESPONSE,
|
||||
"id": "1C3RatW-Jtuc_uMPhaLroA0",
|
||||
"model": "gemini/gemini-3.8-flash",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"I need to inspect `shop/validation.py` next to examine how `validate_quantity` handles zero, "
|
||||
"negative, and non-integer quantities."
|
||||
),
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "call_412671",
|
||||
"name": "read_file",
|
||||
"input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40},
|
||||
"provider_specific_fields": {"signature": GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE},
|
||||
},
|
||||
],
|
||||
"usage": {"input_tokens": 354, "output_tokens": 385},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def with_thinking(case: TranslationTestCase, scenario: str, field: str, value: JsonValue) -> TranslationTestCase:
|
||||
return replace(
|
||||
case,
|
||||
scenario=scenario,
|
||||
litellm_request={**case.litellm_request, "thinking": {**ADAPTIVE_THINKING, field: value}},
|
||||
)
|
||||
|
||||
|
||||
THINKING_ASKS: Final = (
|
||||
with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_summarized", "display", "summarized"),
|
||||
with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_updates", "display", "updates"),
|
||||
with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_int", "display", 7),
|
||||
with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_empty", "display", ""),
|
||||
with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_list", "display", ["omitted"]),
|
||||
with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_long", "display", LONG_SUMMARY),
|
||||
with_thinking(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "display_omitted", "display", "omitted"),
|
||||
with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "summary_auto", "summary", "auto"),
|
||||
with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "summary_empty", "summary", ""),
|
||||
with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "summary_long", "summary", LONG_SUMMARY),
|
||||
with_thinking(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_int", "summary", 7),
|
||||
with_thinking(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_list", "summary", ["auto"]),
|
||||
with_thinking(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_null", "summary", None),
|
||||
)
|
||||
|
||||
|
||||
def test_gemini_keeps_thinking_and_billing_reasoning_without_thought_text_when_the_gateway_turns_it_off(
|
||||
thoughts_off: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
assert_translation(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, thoughts_off, provider)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", THINKING_ASKS, ids=tuple(case.id for case in THINKING_ASKS))
|
||||
def test_a_thinking_display_or_summary_ask_brings_thought_text_back_over_the_gateway_default(
|
||||
case: TranslationTestCase, thoughts_off: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
assert_translation(case, thoughts_off, provider)
|
||||
|
||||
|
||||
def test_display_omitted_alone_keeps_thought_text_on_the_default_gateway(
|
||||
gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
assert_translation(
|
||||
with_thinking(GEMINI_3_8_FLASH_TEST_CASE, "display_omitted_default", "display", "omitted"), gateway, provider
|
||||
)
|
||||
|
||||
|
||||
def test_a_streamed_turn_keeps_thinking_and_billing_reasoning_without_thought_text(
|
||||
thoughts_off: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
provider.expect(
|
||||
Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=tuple(encode_sse_frame(frame) for frame in GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_FRAMES),
|
||||
)
|
||||
)
|
||||
response: Final = thoughts_off.request(
|
||||
"POST", "/v1/messages", {**GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE.litellm_request, "stream": True}
|
||||
)
|
||||
assert_provider_request(
|
||||
provider.received(),
|
||||
"/models/gemini-3.8-flash:streamGenerateContent?alt=sse",
|
||||
GEMINI_3_8_FLASH_TEST_CASE.expected_provider_headers,
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE.expected_provider_request,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
events: Final = tuple(object_value(frame.data) for frame in parse_sse_frames(response.text))
|
||||
started: Final = tuple(
|
||||
object_value(event["content_block"])["type"] for event in events if event.get("type") == "content_block_start"
|
||||
)
|
||||
assert started == ("text", "tool_use")
|
||||
deltas: Final = tuple(
|
||||
object_value(event["delta"]) for event in events if event.get("type") == "content_block_delta"
|
||||
)
|
||||
assert "".join(str(delta.get("text", "")) for delta in deltas) == (
|
||||
"I need to read `shop/validation.py` next to see how `validate_quantity` currently handles zero, negative, "
|
||||
"and non-integer quantities."
|
||||
)
|
||||
assert [object_value(event["usage"]) for event in events if event.get("type") == "message_delta"][-1] == {
|
||||
"input_tokens": 354,
|
||||
"output_tokens": 360,
|
||||
}
|
||||
|
|
@ -0,0 +1,400 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
from unittest.mock import ANY
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, list_value, object_value
|
||||
from integration._support.process import graceful_stop_seconds
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration._support.wire import Reply
|
||||
from integration.translation.case import SSEFrame, TranslationTestCase
|
||||
from integration.translation.gemini_thoughts import LONG_SUMMARY, thoughts_off, with_thinking_config
|
||||
from integration.translation.responses.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE, GEMINI_RESPONSES_RESPONSE
|
||||
from integration.translation.runner import (
|
||||
assert_provider_request,
|
||||
assert_translation,
|
||||
encode_sse_frame,
|
||||
parse_sse_frames,
|
||||
)
|
||||
from pydantic import JsonValue
|
||||
|
||||
__all__ = ("thoughts_off",)
|
||||
|
||||
pytestmark = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE: Final = (
|
||||
"EvQLCvELAWkUfRM8dwxvoWqyX5HLMaxcqWN9o4LPX6M6MZM/OD962KfMSvacFOxwOEb2B3rQM7QgoyT58YrItR4zpDSZJUxA"
|
||||
"N4shekmtkSRpwrqE0rmH1v9AE8w5lR0Jewj8xmiiUXUrK0qvpyxhbVJRmIgdt9CqW+B4bJHYpp1VFAuHMk8eyBRoVNXakpsG"
|
||||
"xjvHWGDomXbyJ7XuG1uHzvA5QtV2lgKUwRNvQqbFgbMhKN4dc8Imm2kwCLQONRN6wYIKJnYIi6ib+MhsFY+hZ6sPysMzVyrk"
|
||||
"nGhc6pZwOKHbUjAraY/X98HO7UEL/q86LKUIvKTS4FfZ8keuyIQQBMdX68fdZPqurR4fI2dr5pFSDbaJWEVaE9XAY9wOXhOE"
|
||||
"uC5SxY0U7nPU39eycNqAoAf+O7i/uHn1eCfLMhvAZyoH0z6vMfofnVn6L6alZtnRFQ3rEdJWEZHFmsLOk7G9SHlLAYKQjEdj"
|
||||
"5aQHsKqBG1SFODPWSeKduI5w/kgFwgrw1ye1rnFTd+FDBs3DsQUlwh+jbPg8tr3KnCL5f2bxoGShUanPcGRAJ4b/IWGM3+wf"
|
||||
"Whcx+3JY/OfuklkCnw86uRlcJiEjWd9XZ+jAby3ECZxBuXsHSE+Ynzpd0mp5g/S1orzqlmUJ/PlOcbDR/AsFduZvZwG0n4hq"
|
||||
"hbXIIPszIpEyGDhkC//vxXsWdo5gLcsnEoUlLCESN3Sc12goKneId6u8pVWfzvHwRa0pK0LIWo4GcWZepZpEoYARlUZN6Dat"
|
||||
"FNJ0nFW1sGaNb4CEYdSSp3m+GVobHpdz8qsGJxWrzzc6ZYmO7LbDJ25dJtyCl9QcfyzGbkFVDdAaTBEM23EdyZMFc2IIT24K"
|
||||
"OVx5esCx+VjDZOmOzue/ZR0xrfaPXVa0CMuzGA8WwiphpWTbD8xZUnqEUZQNwKrjU91+SIhaBKcTVM8F1Wj0t9vlQkkMlzth"
|
||||
"bJxIw+Ccu51gh/s8/awIDsBwC2Ax6eYIhUoWxUWYcphX7r3lBKhIuzv7FyO8tmaRkIOy6Eb1lXHB4fHAIOimZ/n4ClgK/KOZ"
|
||||
"XVLosaGMKsmjYAdrxxPaNqs+say+pQ08bec/c+mqWeDbTOJ2JrCKM/Q+EcyD+82QqWlO68boeJyyt0fIvTCAdn57xN5Th7r1"
|
||||
"WGhT9YHeSB1Y77P9FaJLQqAOjr+yogG7suiCP4QHg1arBlVzvaxAdnlBeXSv6pbQJuuvsFV9+SfwKHVXaBv68bcjSYKm8YZa"
|
||||
"vfK57jweJUxvknCUpzrDRBdVZX28H60eQ3X6S0cV/696AO8NHoKuMv9zsCyu4WXWnpYEObtVD4Sn5hdSn1tnOkyUTLmN+bHV"
|
||||
"sRGe5WTmrgnTKbrGDqpeqFpYLeMftkKWX+DHZj/LvZiNH1pHJid4nbEvcUKHciculasWiLZXDhEDKA1uzvSeG7CHjtHziqRi"
|
||||
"b0dY7pQ0iEJUrjIwMVPnFZoyrywYZVPYgk+jna9FJypscuCz5XzdV+N2b2yrKryU6NhOQ5u/lzMNEFsyYxuXp6LZ5CVgdH1T"
|
||||
"k+1rUf3sWPTO+CyDj+GzWvo5ljP7HihqyitafZJeKOSfaPfrJpBq+R8jnwDOVLCS1lnkhsof12PFoQHAE0YzGDtVscrstW4O"
|
||||
"OaEdtNyNh30fhwwaZ/XGM4O9yJ8l7CgUvrgY/t8/f1hB/ZIc980HGs3G/bmVBYTxeZ18rqanrURtVuNtveyGief6eiEVIFjo"
|
||||
"HWkAFXM+sn1zsJI6l4N9TPMbkoPxLCw2KaOaI/7mQjsH6VktpGuuqpHw3S5tiw3OGwh0lEwydKT0bk4hUhWqoBHqFpCrmKqA"
|
||||
"g3z0kEHxQjQ1j8GmHusdPp4odrCWphw44m1VeWYjF4VjQ16Vik3SSrC8g3v9sHhtKM+3RL6z8zzukqgcQkXf7RXWvmFttckj"
|
||||
"yYWM+vfQm/UvCzQ4dKvFHMqgmwuyEISPO9CD68/518wNpbQD1iB/RnxKwyYoD4u9YQ9TzXCzk2eP0NBB6pRmGg5nBo2SP0PB"
|
||||
"BRZqlxnFX4XfQiS9pxNL"
|
||||
)
|
||||
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE: Final[Mapping[str, JsonValue]] = {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"text": "To see how zero, negative, and non-integer quantities are currently handled and rejected by the validation logic, the next file I need to inspect is `shop/validation.py`.",
|
||||
},
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "read_file",
|
||||
"args": {
|
||||
"path": "shop/validation.py",
|
||||
"start_line": 1,
|
||||
"end_line": 50,
|
||||
},
|
||||
"id": "call_584229",
|
||||
},
|
||||
"thoughtSignature": GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE,
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
"finishMessage": "Model generated function call(s).",
|
||||
},
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 354,
|
||||
"candidatesTokenCount": 70,
|
||||
"totalTokenCount": 790,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": "TEXT",
|
||||
"tokenCount": 354,
|
||||
},
|
||||
],
|
||||
"thoughtsTokenCount": 366,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "2y3RasgqkpyMxw-RxpeQBg",
|
||||
}
|
||||
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_THOUGHT_SIGNATURE: Final = (
|
||||
"EvYSCvMSAWkUfROtzl1tHuKrIElCi7xsI6n+7cPuqsM/qZSYIpK3gONKCCo0E4moadzMQ3sN09ll3gRjbcBg6rKSck+JTBhd"
|
||||
"mrLsP/sqfxiYfMx4lWzDeq/Pw/IPhT2g5/2EvSU4FDsUczorTsD55gKvgRT8v05YTFMGF/np8r3VT52Q2yAzwRK+31qn8pYS"
|
||||
"c25if5coTZJrvmJZDlZ9/64aibghRps9zN07yRVXpUjtLyKMaTDH0loGgiQn0I7Cxfhw9oUyZels7Un7pB7omA4RKW5xp5rx"
|
||||
"Y+WgSeAEyyxNcPIkRvaxHaY1p2QaR3m17a0L7joJYNB+gNg7R/4Mfwpr8WuSOyLfBaKTuC5IiYWLaRm0ibASYzd59pPUyArt"
|
||||
"mIx7pPfWxAr8/FT8TVQTSSeHXF88x/xJ2WSTSzQ3+FPw6Wm/LpfeGwQPX/g1+8f+KVcj1qAwT/tUobqOh7DdWl68CnNFReX4"
|
||||
"SwBqRjTzZ276P7l15fhWmnQkiWXmLVsufJi5VXidEVrAsuEyVe7JmpExi3Y+0pn82D9x7lyv/TrOCVQgKwC0Zoq9NfuQxWa0"
|
||||
"RvqmAv6toMfI8owTyzVoItnoGdhcYamwr7AyECKTuqnzMuylzOx5ttZnRc12/Ey2GQR3UxhyxMNLk01/UDicLnEXFl8Zn56i"
|
||||
"rj51atVum7mJc48qekcKXQ5Lc3/t277v1d+eLmuiHebshpQwg+qQPHnnfky0WL98w343XDb29wwK3+veqQSjrffv5rQY9Tvq"
|
||||
"3PMuN1pjU7Z3rL6fZmt5UrhqQQnTf4dA7X7n/aLkpA8SpY6ICDDr2x1z0yW5kMnHfxxk+N+j5eWwoTJuxxy94rLaYj3CKT57"
|
||||
"XhMthQAoF1DGNpGxhbfC/tVuFE9FkxEBiYXqDz5fy2AU+S2zatT7/9mkFaHLY6hkOCS48X9WBVobxsQZZUnkbxplCzg4Huin"
|
||||
"gPnl1bUDEprLHfTYgZHcnn3FgWFPwzxNBebrc4xjqXciMgReu4kOLP18JZbIJSjhP/RZAnQJEEOI8OrVGZlmiC7/JJx+8PTX"
|
||||
"XYMTKtiVU/R3gOhOeVNN9WXruUoQzuy+JfJGg8ZQ2ni0Ugkx0sLO5rDsTOZ67bA+B99FQnQcBLyb40EPTOzrLrcM9wzXL+YZ"
|
||||
"JUwNtdaQGxh/aUeKex/8of7GUH9ZsCZW0YN5xuEs8e2Q+vi2Msluckvki/wcyx7oOxmjw8bEbUkPBFq/ZEOcQqtetPLBzNPt"
|
||||
"smJ4EfOU/kwBBLxplt6JrF+0ILeWJuH7OGpvpseXEosLpJ9HV4oIHL9co8EWGH/oAUzcuvhymR8FRziVE2IgVkreryIfHAs/"
|
||||
"umsJgk1dXZk4A/R7KtjHKau0yxLAYJh6jWLvLsLv5N44VfKCQtyWFdUOSGbqn6E4YuOYVc+x+TGQcSMniobTekwm+UtqzVH8"
|
||||
"mmtd8nwzwVkDgqZEAHrrnJOALLPeIbPuQ0/zjuUJmQovnnSqUyyCxQDpaxrlLD3iIZp9/9hPTY+lYHlam7VfChpQ7dPQ075t"
|
||||
"ECiPA5TgMzh1qNQBF+rgKtrcrPo1ZDFqlseAybmSKqnCdvD5vdfdeVL8cmQIQao9qHDVh/aR0uwOsVKKzavPIYH2mhomg/cz"
|
||||
"XB1nJG2zXB99g08Bd14c/zHHPNeqkqmwURCTxzPh8aeoRPcC8uL/eyawL66FfyUo40og9uridBDA4qx4ZMI+P1dBKFnMPM7o"
|
||||
"Blpo+Sg9m6XZNPNVbDqRKKmX9OWCng7/PAKX+wa8EzRetNbyh73eMviMZaiN9YAyoKiS+nR5FqrPgNcFMqa8dHDEyY6O3Vuh"
|
||||
"5Er5R3iGNACXMQvLRDVy/1UmFY1mzwew9lrDcUkZ0MNKBrbtDC87XEFGMejqvoJgF5Yl+q1mJE6gMBDTo6UEP1SOctxHyX23"
|
||||
"mvqhCK9wOvK91UGnNyd+8ZQtQPP/L99d2y7hHmE6VEA/nUa2qkkO99vEQmJV3ilax+wZknABwk5sFdh7vWLwhEvxkdGYUh6c"
|
||||
"HnA70VX5Kh0twJPKwE7gNASf/rvE5kYdrei5RtZNKMpVulXMPVho3eRBW6Qf6Ka0JjDZF5Dr8IXXsjP2LFTmnLNcnxu0FXx8"
|
||||
"4xRuw1+yN3cP5R8LFlSGJmqPf5kFwAwpzT5FAXETiBaj7lg3WyG+OaDpab2NG+BvHbdfrDAuFKSsTDZ3Vt58fj6fuhS8tFd4"
|
||||
"YzWL6X30pj+r7QRAefk+2CQDKM1TnbAJbyBcG2DVfeWpgmNjtzoV+RXNykIFIy6rSeOnUBAJCXwbAe7Scp5w/lx03a5aooRU"
|
||||
"CP7x7CAmAU+YuGerG7A/3bbWDqN4EHGawkloK9UXmlOVX7uzLy0q3fJZb/6pSvPX+DR2rjFq0CSKnNFyBXYQRHx+DekAHWJS"
|
||||
"VGUw7RLAPd4DE2HRiYTss7vtXalkP0hQ/vnt8lS9bjyBCi8ly+4FjFpo/SeKx0TKFulbFp1GPpsNERT+7kEkTsNZQHXWXPRB"
|
||||
"PH0qmpFWZjI98fzWIGXGF376X0bcg6g67C/K/+YtH+QIU82ScAM0ozzQ1uzb0ci6wnM4JjNKA3ctVX6zuPR20Tb7VLlq5rwR"
|
||||
"coOKjGQmBGbUiW5aKAAsMnIDhxf2lPCcwwb51C0Zb6hfOG+ka7LLcstgq4fmXHQ11BbTFu9ctAhy0E5V3aYSJAilb6VXZdD4"
|
||||
"kSl3I0D2Xc/pTLPAzsxg6mEHL7J/DD1lRvEtSh4U34RCwxMk+pQbfEo6y/lkIVhXuE8p7d4mhPtsDJkls/3vnl+rR5ycebjH"
|
||||
"g/zC01CjHhbVglb+HVEC5FMTrBNy/bweTE6c7qNHV6EWRN64nqvs4IW/jEVoaI2qt9WAJZ1br21rCE01iQagQUs1/ZH2KpQB"
|
||||
"jTe3ju/BotwiE4TdqHcDL/VP62yq9KLeiBNXY2ctb3NwJzzfLrM2KHv2b4YmWuZ/VLABBfh9r3OzSmnFBvmWsZjZOHHEQXMI"
|
||||
"hI+WDpRGB54CWmnWs6YzQK70IKrZUtq4BfPQJGf8OqO/ZeCJA4kinQbCaTTqQOJLsONKbPTqm8ImXz/QiML+KFvY3Z3ReW3T"
|
||||
"NsJ3CGW7NRY1cfBIkhHuKHxPK1Ua5H3b7yP+h8OKrbqyjYPw2kE56TOH75HnIp/Ib6w+7uZpMVIbM4fYCRWlvQ7cUtaRq4fk"
|
||||
"qqgHykE8bJYoU9eJFmPJNj4gt051sXJoslr2C6mvHYdSZgKxNa1ksd95cPKS3yURYQ=="
|
||||
)
|
||||
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_FRAMES: Final = (
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"text": "To understand how zero, negative, and non-integer quantities are checked during validation, I need to inspect",
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"index": 0,
|
||||
},
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 354,
|
||||
"candidatesTokenCount": 21,
|
||||
"totalTokenCount": 847,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": "TEXT",
|
||||
"tokenCount": 354,
|
||||
},
|
||||
],
|
||||
"thoughtsTokenCount": 472,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "3S3Rat6pNonj_uMPv9i78Qw",
|
||||
}
|
||||
),
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"text": " `shop/validation.py` next.",
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"index": 0,
|
||||
},
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 354,
|
||||
"candidatesTokenCount": 30,
|
||||
"totalTokenCount": 856,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": "TEXT",
|
||||
"tokenCount": 354,
|
||||
},
|
||||
],
|
||||
"thoughtsTokenCount": 472,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "3S3Rat6pNonj_uMPv9i78Qw",
|
||||
}
|
||||
),
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"functionCall": {
|
||||
"name": "read_file",
|
||||
"args": {
|
||||
"end_line": 50,
|
||||
"path": "shop/validation.py",
|
||||
"start_line": 1,
|
||||
},
|
||||
"id": "call_5018",
|
||||
},
|
||||
"thoughtSignature": GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_THOUGHT_SIGNATURE,
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"index": 0,
|
||||
},
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 354,
|
||||
"candidatesTokenCount": 63,
|
||||
"totalTokenCount": 889,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": "TEXT",
|
||||
"tokenCount": 354,
|
||||
},
|
||||
],
|
||||
"thoughtsTokenCount": 472,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "3S3Rat6pNonj_uMPv9i78Qw",
|
||||
}
|
||||
),
|
||||
SSEFrame(
|
||||
data={
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"text": "",
|
||||
},
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
},
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 354,
|
||||
"candidatesTokenCount": 63,
|
||||
"totalTokenCount": 889,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": "TEXT",
|
||||
"tokenCount": 354,
|
||||
},
|
||||
],
|
||||
"thoughtsTokenCount": 472,
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
"responseId": "3S3Rat6pNonj_uMPv9i78Qw",
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE: Final = replace(
|
||||
GEMINI_3_8_FLASH_TEST_CASE,
|
||||
scenario="thoughts_off",
|
||||
expected_provider_request=with_thinking_config(
|
||||
GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, {"thinkingLevel": "high", "includeThoughts": False}
|
||||
),
|
||||
mock_provider_response=GEMINI_3_8_FLASH_THOUGHTS_OFF_RESPONSE,
|
||||
expected_litellm_response={
|
||||
**GEMINI_RESPONSES_RESPONSE,
|
||||
"model": "gemini/gemini-3.8-flash",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": ANY,
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": (
|
||||
"To see how zero, negative, and non-integer quantities are currently handled and rejected "
|
||||
"by the validation logic, the next file I need to inspect is `shop/validation.py`."
|
||||
),
|
||||
"annotations": [],
|
||||
},
|
||||
],
|
||||
"phase": None,
|
||||
},
|
||||
{
|
||||
"arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}',
|
||||
"call_id": "call_584229__thought__" + GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE,
|
||||
"name": "read_file",
|
||||
"type": "function_call",
|
||||
"id": "fc_call_584229__thought__" + GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE,
|
||||
"namespace": None,
|
||||
"status": "completed",
|
||||
"provider_specific_fields": {"thought_signature": GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE},
|
||||
},
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 354,
|
||||
"input_tokens_details": {
|
||||
"audio_tokens": None,
|
||||
"cached_tokens": 0,
|
||||
"cached_tokens_details": None,
|
||||
"image_tokens": None,
|
||||
"text_tokens": 354,
|
||||
"video_tokens": None,
|
||||
},
|
||||
"output_tokens": 436,
|
||||
"output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 366, "text_tokens": 70},
|
||||
"total_tokens": 790,
|
||||
"cost": None,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def with_summary(case: TranslationTestCase, scenario: str, summary: JsonValue) -> TranslationTestCase:
|
||||
reasoning: Final[Mapping[str, JsonValue]] = {"effort": "high", "summary": summary}
|
||||
return replace(
|
||||
case,
|
||||
scenario=scenario,
|
||||
litellm_request={**case.litellm_request, "reasoning": dict(reasoning)},
|
||||
expected_litellm_response={**case.expected_litellm_response, "reasoning": dict(reasoning)},
|
||||
)
|
||||
|
||||
|
||||
SUMMARY_ASKS: Final = (
|
||||
with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_auto", "auto"),
|
||||
with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_concise", "concise"),
|
||||
with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_empty", ""),
|
||||
with_summary(GEMINI_3_8_FLASH_TEST_CASE, "summary_long", LONG_SUMMARY),
|
||||
with_summary(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_int", 7),
|
||||
with_summary(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_list", ["auto"]),
|
||||
with_summary(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, "summary_null", None),
|
||||
)
|
||||
|
||||
|
||||
def test_gemini_keeps_thinking_and_billing_reasoning_without_thought_text_when_the_gateway_turns_it_off(
|
||||
thoughts_off: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
assert_translation(GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE, thoughts_off, provider)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", SUMMARY_ASKS, ids=tuple(case.id for case in SUMMARY_ASKS))
|
||||
def test_a_reasoning_summary_ask_brings_thought_text_back_over_the_gateway_default(
|
||||
case: TranslationTestCase, thoughts_off: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
assert_translation(case, thoughts_off, provider)
|
||||
|
||||
|
||||
def test_a_streamed_turn_keeps_thinking_and_billing_reasoning_without_thought_text(
|
||||
thoughts_off: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
provider.expect(
|
||||
Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=tuple(encode_sse_frame(frame) for frame in GEMINI_3_8_FLASH_THOUGHTS_OFF_STREAM_FRAMES),
|
||||
)
|
||||
)
|
||||
response: Final = thoughts_off.request(
|
||||
"POST", "/v1/responses", {**GEMINI_3_8_FLASH_TEST_CASE.litellm_request, "stream": True}
|
||||
)
|
||||
assert_provider_request(
|
||||
provider.received(),
|
||||
"/models/gemini-3.8-flash:streamGenerateContent?alt=sse",
|
||||
GEMINI_3_8_FLASH_TEST_CASE.expected_provider_headers,
|
||||
GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE.expected_provider_request,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
events: Final = tuple(
|
||||
object_value(frame.data) for frame in parse_sse_frames(response.text) if frame.data != "[DONE]"
|
||||
)
|
||||
assert [event["type"] for event in events if str(event["type"]).startswith("response.reasoning")] == []
|
||||
assert "".join(str(event["delta"]) for event in events if event["type"] == "response.output_text.delta") == (
|
||||
"To understand how zero, negative, and non-integer quantities are checked during validation, I need to "
|
||||
"inspect `shop/validation.py` next."
|
||||
)
|
||||
completed: Final = object_value(
|
||||
next(event["response"] for event in events if event["type"] == "response.completed")
|
||||
)
|
||||
assert [object_value(item)["type"] for item in list_value(completed["output"])] == ["message", "function_call"]
|
||||
assert object_value(completed["usage"])["output_tokens_details"] == {
|
||||
"reasoning_tokens": 472,
|
||||
"text_tokens": 63,
|
||||
}
|
||||
|
|
@ -291,6 +291,37 @@ def test_exact_cache_key_includes_anthropic_messages_params(anthropic_param):
|
|||
assert baseline != cache.get_cache_key(model="claude-sonnet-4-5", messages=messages, **anthropic_param)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"summary_ask",
|
||||
[
|
||||
{"reasoning_summary": "auto"},
|
||||
{"reasoningSummary": "auto"},
|
||||
{"extra_body": {"reasoning_summary": "auto"}},
|
||||
{"extra_body": {"reasoningSummary": "auto"}},
|
||||
],
|
||||
ids=["top_level", "top_level_camel", "extra_body", "extra_body_camel"],
|
||||
)
|
||||
def test_exact_cache_key_includes_the_reasoning_summary_alias(summary_ask: dict[str, object]) -> None:
|
||||
"""The AI SDK summary alias decides whether thought summaries come back, wherever the request
|
||||
carries it, so a reply cached without it must not answer a request that asks for one."""
|
||||
cache: Final = Cache(type=LiteLLMCacheType.LOCAL)
|
||||
messages: Final = [{"role": "user", "content": "how much is the ball?"}]
|
||||
no_ask: Final = cache.get_cache_key(model="gemini/gemini-3.8-flash", messages=messages, reasoning_effort="medium")
|
||||
ask: Final = cache.get_cache_key(
|
||||
model="gemini/gemini-3.8-flash", messages=messages, reasoning_effort="medium", **summary_ask
|
||||
)
|
||||
assert no_ask != ask
|
||||
assert ask == cache.get_cache_key(
|
||||
model="gemini/gemini-3.8-flash", messages=messages, reasoning_effort="medium", **summary_ask
|
||||
)
|
||||
assert ask != cache.get_cache_key(
|
||||
model="gemini/gemini-3.8-flash",
|
||||
messages=messages,
|
||||
reasoning_effort="medium",
|
||||
reasoning_summary="detailed",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_cache_skips_write_when_one_input_yields_many_embeddings(monkeypatch):
|
||||
"""A cross-encoder behind /embeddings returns one score per document for a single
|
||||
|
|
@ -561,10 +592,7 @@ def test_caching_v2(): # test in memory cache
|
|||
litellm.cache = None # disable cache
|
||||
litellm.success_callback = []
|
||||
litellm._async_success_callback = []
|
||||
if (
|
||||
response2["choices"][0]["message"]["content"]
|
||||
!= response1["choices"][0]["message"]["content"]
|
||||
):
|
||||
if response2["choices"][0]["message"]["content"] != response1["choices"][0]["message"]["content"]:
|
||||
print(f"response1: {response1}")
|
||||
print(f"response2: {response2}")
|
||||
pytest.fail(f"Error occurred:")
|
||||
|
|
@ -595,10 +623,7 @@ def test_caching_with_ttl():
|
|||
litellm.cache = None # disable cache
|
||||
litellm.success_callback = []
|
||||
litellm._async_success_callback = []
|
||||
assert (
|
||||
response2["choices"][0]["message"]["content"]
|
||||
!= response1["choices"][0]["message"]["content"]
|
||||
)
|
||||
assert response2["choices"][0]["message"]["content"] != response1["choices"][0]["message"]["content"]
|
||||
except Exception as e:
|
||||
print(f"error occurred: {traceback.format_exc()}")
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
|
@ -632,9 +657,7 @@ def test_caching_with_default_ttl():
|
|||
|
||||
|
||||
def test_caching_with_models_v2():
|
||||
messages = [
|
||||
{"role": "user", "content": "who is ishaan CTO of litellm from litellm 2023"}
|
||||
]
|
||||
messages = [{"role": "user", "content": "who is ishaan CTO of litellm from litellm 2023"}]
|
||||
litellm.cache = Cache()
|
||||
print("test2 for caching")
|
||||
litellm.set_verbose = True
|
||||
|
|
@ -657,18 +680,12 @@ def test_caching_with_models_v2():
|
|||
litellm.cache = None
|
||||
litellm.success_callback = []
|
||||
litellm._async_success_callback = []
|
||||
if (
|
||||
response3["choices"][0]["message"]["content"]
|
||||
== response2["choices"][0]["message"]["content"]
|
||||
):
|
||||
if response3["choices"][0]["message"]["content"] == response2["choices"][0]["message"]["content"]:
|
||||
# if models are different, it should not return cached response
|
||||
print(f"response2: {response2}")
|
||||
print(f"response3: {response3}")
|
||||
pytest.fail(f"Error occurred:")
|
||||
if (
|
||||
response1["choices"][0]["message"]["content"]
|
||||
!= response2["choices"][0]["message"]["content"]
|
||||
):
|
||||
if response1["choices"][0]["message"]["content"] != response2["choices"][0]["message"]["content"]:
|
||||
print(f"response1: {response1}")
|
||||
print(f"response2: {response2}")
|
||||
pytest.fail(f"Error occurred:")
|
||||
|
|
@ -708,9 +725,7 @@ def test_get_cache_key():
|
|||
cache_key = cache_instance.get_cache_key(
|
||||
**{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [
|
||||
{"role": "user", "content": "write a one sentence poem about: 7510"}
|
||||
],
|
||||
"messages": [{"role": "user", "content": "write a one sentence poem about: 7510"}],
|
||||
"max_tokens": 40,
|
||||
"temperature": 0.2,
|
||||
"stream": True,
|
||||
|
|
@ -721,9 +736,7 @@ def test_get_cache_key():
|
|||
cache_key_2 = cache_instance.get_cache_key(
|
||||
**{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [
|
||||
{"role": "user", "content": "write a one sentence poem about: 7510"}
|
||||
],
|
||||
"messages": [{"role": "user", "content": "write a one sentence poem about: 7510"}],
|
||||
"max_tokens": 40,
|
||||
"temperature": 0.2,
|
||||
"stream": True,
|
||||
|
|
@ -736,9 +749,9 @@ def test_get_cache_key():
|
|||
# Hexadecimal representation of the hash
|
||||
hash_hex = hash_object.hexdigest()
|
||||
assert cache_key == hash_hex
|
||||
assert (
|
||||
cache_key_2 == hash_hex
|
||||
), f"{cache_key} != {cache_key_2}. The same kwargs should have the same cache key across runs"
|
||||
assert cache_key_2 == hash_hex, (
|
||||
f"{cache_key} != {cache_key_2}. The same kwargs should have the same cache key across runs"
|
||||
)
|
||||
|
||||
embedding_cache_key = cache_instance.get_cache_key(
|
||||
**{
|
||||
|
|
@ -756,15 +769,13 @@ def test_get_cache_key():
|
|||
|
||||
print(embedding_cache_key)
|
||||
|
||||
embedding_cache_key_str = (
|
||||
"model: azure/text-embedding-ada-002input: ['hi who is ishaan']"
|
||||
)
|
||||
embedding_cache_key_str = "model: azure/text-embedding-ada-002input: ['hi who is ishaan']"
|
||||
hash_object = hashlib.sha256(embedding_cache_key_str.encode())
|
||||
# Hexadecimal representation of the hash
|
||||
hash_hex = hash_object.hexdigest()
|
||||
assert (
|
||||
embedding_cache_key == hash_hex
|
||||
), f"{embedding_cache_key} != 'model: azure/text-embedding-ada-002input: ['hi who is ishaan']'. The same kwargs should have the same cache key across runs"
|
||||
assert embedding_cache_key == hash_hex, (
|
||||
f"{embedding_cache_key} != 'model: azure/text-embedding-ada-002input: ['hi who is ishaan']'. The same kwargs should have the same cache key across runs"
|
||||
)
|
||||
|
||||
# Proxy - embedding cache, test if embedding key, gets model_group and not model
|
||||
embedding_cache_key_2 = cache_instance.get_cache_key(
|
||||
|
|
@ -817,9 +828,7 @@ def test_get_cache_key():
|
|||
)
|
||||
|
||||
print(embedding_cache_key_2)
|
||||
embedding_cache_key_str_2 = (
|
||||
"model: EMBEDDING_MODEL_GROUPinput: ['hi who is ishaan']"
|
||||
)
|
||||
embedding_cache_key_str_2 = "model: EMBEDDING_MODEL_GROUPinput: ['hi who is ishaan']"
|
||||
hash_object = hashlib.sha256(embedding_cache_key_str_2.encode())
|
||||
# Hexadecimal representation of the hash
|
||||
hash_hex = hash_object.hexdigest()
|
||||
|
|
@ -852,9 +861,7 @@ def test_redis_caching_multiple_namespaces():
|
|||
# Mock the Redis client creation from the _redis module
|
||||
with (
|
||||
patch("litellm._redis.get_redis_client") as mock_get_redis_client,
|
||||
patch(
|
||||
"litellm._redis.get_redis_connection_pool"
|
||||
) as mock_get_redis_connection_pool,
|
||||
patch("litellm._redis.get_redis_connection_pool") as mock_get_redis_connection_pool,
|
||||
):
|
||||
# Create a mock Redis client that simulates real Redis behavior
|
||||
mock_redis_client = MagicMock()
|
||||
|
|
@ -927,18 +934,10 @@ def test_redis_caching_multiple_namespaces():
|
|||
mock_response="Response without namespace",
|
||||
)
|
||||
|
||||
print(
|
||||
f"Response 1 type: {type(response_1)} - ID: {getattr(response_1, 'id', 'N/A')}"
|
||||
)
|
||||
print(
|
||||
f"Response 2 type: {type(response_2)} - ID: {getattr(response_2, 'id', 'N/A')}"
|
||||
)
|
||||
print(
|
||||
f"Response 3 type: {type(response_3)} - Cache hit: {isinstance(response_3, str)}"
|
||||
)
|
||||
print(
|
||||
f"Response 4 type: {type(response_4)} - ID: {getattr(response_4, 'id', 'N/A')}"
|
||||
)
|
||||
print(f"Response 1 type: {type(response_1)} - ID: {getattr(response_1, 'id', 'N/A')}")
|
||||
print(f"Response 2 type: {type(response_2)} - ID: {getattr(response_2, 'id', 'N/A')}")
|
||||
print(f"Response 3 type: {type(response_3)} - Cache hit: {isinstance(response_3, str)}")
|
||||
print(f"Response 4 type: {type(response_4)} - ID: {getattr(response_4, 'id', 'N/A')}")
|
||||
|
||||
print(f"Redis storage keys: {list(redis_storage.keys())}")
|
||||
|
||||
|
|
@ -947,10 +946,7 @@ def test_redis_caching_multiple_namespaces():
|
|||
namespace_1_keys = [k for k in cache_keys if k.startswith(f"{namespace_1}:")]
|
||||
namespace_2_keys = [k for k in cache_keys if k.startswith(f"{namespace_2}:")]
|
||||
no_namespace_keys = [
|
||||
k
|
||||
for k in cache_keys
|
||||
if not k.startswith(f"{namespace_1}:")
|
||||
and not k.startswith(f"{namespace_2}:")
|
||||
k for k in cache_keys if not k.startswith(f"{namespace_1}:") and not k.startswith(f"{namespace_2}:")
|
||||
]
|
||||
|
||||
print(f"Namespace 1 keys: {namespace_1_keys}")
|
||||
|
|
@ -963,9 +959,7 @@ def test_redis_caching_multiple_namespaces():
|
|||
assert len(no_namespace_keys) > 0, "Should have cache keys for no namespace"
|
||||
|
||||
# The main test: response 3 should be a cache hit (string) because it uses same namespace as response 1
|
||||
assert isinstance(
|
||||
response_3, str
|
||||
), "Response 3 should be a cache hit (string) for same namespace"
|
||||
assert isinstance(response_3, str), "Response 3 should be a cache hit (string) for same namespace"
|
||||
|
||||
# response 1 & 2 should be ModelResponse objects (cache misses)
|
||||
assert hasattr(response_1, "id"), "Response 1 should be a ModelResponse object"
|
||||
|
|
@ -973,14 +967,14 @@ def test_redis_caching_multiple_namespaces():
|
|||
assert hasattr(response_4, "id"), "Response 4 should be a ModelResponse object"
|
||||
|
||||
# response 1 & 2 should have different IDs (different namespaces)
|
||||
assert (
|
||||
response_1.id != response_2.id
|
||||
), f"Expected different response ID for different namespace. Got {response_1.id} and {response_2.id}"
|
||||
assert response_1.id != response_2.id, (
|
||||
f"Expected different response ID for different namespace. Got {response_1.id} and {response_2.id}"
|
||||
)
|
||||
|
||||
# response 1 & 4 should have different IDs (different namespaces)
|
||||
assert (
|
||||
response_1.id != response_4.id
|
||||
), f"Expected different response ID for no namespace vs namespaced. Got {response_1.id} and {response_4.id}"
|
||||
assert response_1.id != response_4.id, (
|
||||
f"Expected different response ID for no namespace vs namespaced. Got {response_1.id} and {response_4.id}"
|
||||
)
|
||||
|
||||
|
||||
_TOOL_TURN_ITEM: Final = {"role": "user", "content": "hi"}
|
||||
|
|
@ -1305,12 +1299,8 @@ async def test_embedding_caching_individual_items(monkeypatch: pytest.MonkeyPatc
|
|||
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL))
|
||||
item: Final = f"individual embedding {uuid.uuid4().hex}"
|
||||
|
||||
first: Final = await aembedding(
|
||||
model="text-embedding-3-small", input=item, caching=True, mock_response="0.1,0.2"
|
||||
)
|
||||
second: Final = await aembedding(
|
||||
model="text-embedding-3-small", input=item, caching=True, mock_response="0.3,0.4"
|
||||
)
|
||||
first: Final = await aembedding(model="text-embedding-3-small", input=item, caching=True, mock_response="0.1,0.2")
|
||||
second: Final = await aembedding(model="text-embedding-3-small", input=item, caching=True, mock_response="0.3,0.4")
|
||||
|
||||
assert second.data[0].embedding == first.data[0].embedding
|
||||
assert second._hidden_params["cache_hit"] is True
|
||||
|
|
@ -1448,9 +1438,7 @@ async def test_caching_with_cache_controls(monkeypatch: pytest.MonkeyPatch) -> N
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_caching_redis_ttl(
|
||||
monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None
|
||||
) -> None:
|
||||
async def test_embedding_caching_redis_ttl(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None:
|
||||
mock_pipeline: Final = AsyncMock()
|
||||
mock_set: Final = AsyncMock()
|
||||
mock_pipeline.__aenter__.return_value.set = mock_set
|
||||
|
|
@ -1506,18 +1494,14 @@ async def test_logging_turn_off_message_logging_streaming(
|
|||
assert cached_response["choices"][0]["message"]["content"] == "cached"
|
||||
|
||||
|
||||
def test_redis_caching_default_ttl(
|
||||
monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None
|
||||
) -> None:
|
||||
def test_redis_caching_default_ttl(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None:
|
||||
monkeypatch.setattr(litellm, "default_redis_ttl", 120)
|
||||
cache: Final = RedisCache(host="127.0.0.1", port="6379")
|
||||
|
||||
assert cache.default_ttl == 120
|
||||
|
||||
|
||||
def test_redis_caching_llm_caching_ttl(
|
||||
monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None
|
||||
) -> None:
|
||||
def test_redis_caching_llm_caching_ttl(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None:
|
||||
monkeypatch.setattr(litellm, "default_redis_ttl", 120)
|
||||
cache: Final = RedisCache(host="127.0.0.1", port="6379")
|
||||
set_cache: Final = MagicMock()
|
||||
|
|
@ -1529,9 +1513,7 @@ def test_redis_caching_llm_caching_ttl(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_redis_caching_ttl_pipeline(
|
||||
monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None
|
||||
) -> None:
|
||||
async def test_redis_caching_ttl_pipeline(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None:
|
||||
monkeypatch.setattr(litellm, "default_redis_ttl", 120)
|
||||
cache: Final = RedisCache(host="127.0.0.1", port="6379")
|
||||
pipeline: Final = AsyncMock()
|
||||
|
|
@ -1551,9 +1533,7 @@ async def test_redis_caching_ttl_pipeline(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_redis_caching_ttl_sadd(
|
||||
monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None
|
||||
) -> None:
|
||||
async def test_redis_caching_ttl_sadd(monkeypatch: pytest.MonkeyPatch, redis_client_without_network: None) -> None:
|
||||
monkeypatch.setattr(litellm, "default_redis_ttl", 120)
|
||||
cache: Final = RedisCache(host="127.0.0.1", port="6379")
|
||||
redis_client: Final = AsyncMock()
|
||||
|
|
|
|||
|
|
@ -6,13 +6,17 @@ regression they guard is the one a caller sees: a tier the proxy advertises has
|
|||
leaves the adapter, in the shape the target expects.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.pass_through.adapters.handler import (
|
||||
LiteLLMMessagesToCompletionTransformationHandler,
|
||||
)
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
|
||||
from litellm.utils import reasoning_effort_with_summary_alias
|
||||
|
||||
MESSAGES = [{"role": "user", "content": "hello"}]
|
||||
|
||||
|
|
@ -38,14 +42,14 @@ def _reasoning_effort_sent(model: str, provider: str, reasoning_effort: object)
|
|||
return completion_kwargs.get("reasoning_effort")
|
||||
|
||||
|
||||
def _reasoning_effort_sent_for_thinking(
|
||||
def _completion_kwargs_for_thinking(
|
||||
model: str,
|
||||
provider: str | None,
|
||||
thinking: dict[str, object],
|
||||
*,
|
||||
tools: list[dict[str, object]] | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> object:
|
||||
) -> Mapping[str, object]:
|
||||
extra_kwargs: Final = {
|
||||
key: value for key, value in (("custom_llm_provider", provider), ("api_base", api_base)) if value is not None
|
||||
}
|
||||
|
|
@ -66,7 +70,20 @@ def _reasoning_effort_sent_for_thinking(
|
|||
output_format=None,
|
||||
extra_kwargs=extra_kwargs,
|
||||
)
|
||||
return completion_kwargs.get("reasoning_effort")
|
||||
return completion_kwargs
|
||||
|
||||
|
||||
def _reasoning_effort_sent_for_thinking(
|
||||
model: str,
|
||||
provider: str | None,
|
||||
thinking: dict[str, object],
|
||||
*,
|
||||
tools: list[dict[str, object]] | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> object:
|
||||
return _completion_kwargs_for_thinking(model, provider, thinking, tools=tools, api_base=api_base).get(
|
||||
"reasoning_effort"
|
||||
)
|
||||
|
||||
|
||||
SUMMARIZED_THINKING = {"type": "enabled", "budget_tokens": 4096, "summary": "auto"}
|
||||
|
|
@ -78,6 +95,66 @@ MULTIPLY_TOOL = {
|
|||
}
|
||||
|
||||
|
||||
class TestAThinkingSummaryAskReachesGemini:
|
||||
"""Gemini maps a summary ask to ``includeThoughts``, so ``gemini_include_thoughts_default`` only
|
||||
decides for a ``/v1/messages`` request that asks for nothing, the same as on chat completions."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, provider",
|
||||
[("gemini/gemini-3.1-pro-preview", "gemini"), ("vertex_ai/gemini-2.5-flash", "vertex_ai")],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"thinking, sends_thoughts",
|
||||
[
|
||||
(SUMMARIZED_THINKING, True),
|
||||
({**PLAIN_THINKING, "display": "summarized"}, True),
|
||||
({**PLAIN_THINKING, "display": "updates"}, True),
|
||||
({**PLAIN_THINKING, "display": "omitted"}, False),
|
||||
(PLAIN_THINKING, False),
|
||||
({"type": "disabled", "display": "summarized"}, False),
|
||||
],
|
||||
)
|
||||
def test_the_request_ask_beats_the_gateway_default(
|
||||
self,
|
||||
local_model_cost_map: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
provider: str,
|
||||
thinking: dict[str, object],
|
||||
sends_thoughts: bool,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "gemini_include_thoughts_default", False)
|
||||
|
||||
sent: Final = _completion_kwargs_for_thinking(model, provider, thinking)
|
||||
gemini: Final = VertexGeminiConfig()
|
||||
reasoning_effort: Final = reasoning_effort_with_summary_alias(
|
||||
sent.get("reasoning_effort"), sent, accepts_summary=gemini.accepts_reasoning_summary_in_reasoning_effort
|
||||
)
|
||||
mapped: Final = gemini.map_openai_params(
|
||||
{"reasoning_effort": reasoning_effort}, {}, model.split("/", 1)[1], True
|
||||
)
|
||||
|
||||
assert mapped["thinkingConfig"]["includeThoughts"] is sends_thoughts
|
||||
assert not isinstance(sent.get("reasoning_effort"), dict)
|
||||
|
||||
def test_auto_summary_leaves_the_gateway_default_in_charge(
|
||||
self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "gemini_include_thoughts_default", False)
|
||||
monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true")
|
||||
|
||||
sent: Final = _reasoning_effort_sent_for_thinking("gemini/gemini-3.1-pro-preview", "gemini", PLAIN_THINKING)
|
||||
|
||||
assert sent == "high"
|
||||
|
||||
def test_a_display_ask_leaves_other_chat_targets_on_the_plain_tier(self, local_model_cost_map: None) -> None:
|
||||
sent: Final = _reasoning_effort_sent_for_thinking(
|
||||
"databricks/databricks-qwen35-122b-a10b", "databricks", {**PLAIN_THINKING, "display": "summarized"}
|
||||
)
|
||||
|
||||
assert sent == "high"
|
||||
|
||||
|
||||
class TestTheSummaryWrappingOnlyRidesTheResponsesBridge:
|
||||
"""Only the Responses API takes ``reasoning_effort`` as a dict. Databricks answered the wrapped
|
||||
``{"effort", "summary"}`` with ``field 'reasoning_effort' expects input with json type 'string'
|
||||
|
|
@ -100,7 +177,9 @@ class TestTheSummaryWrappingOnlyRidesTheResponsesBridge:
|
|||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true")
|
||||
|
||||
sent = _reasoning_effort_sent_for_thinking("databricks/databricks-qwen35-122b-a10b", "databricks", PLAIN_THINKING)
|
||||
sent: Final = _reasoning_effort_sent_for_thinking(
|
||||
"databricks/databricks-qwen35-122b-a10b", "databricks", PLAIN_THINKING
|
||||
)
|
||||
|
||||
assert sent == "high"
|
||||
|
||||
|
|
@ -113,7 +192,7 @@ class TestTheSummaryWrappingOnlyRidesTheResponsesBridge:
|
|||
],
|
||||
)
|
||||
def test_a_bridged_target_keeps_the_summary(self, local_model_cost_map: None, model: str, provider: str) -> None:
|
||||
sent = _reasoning_effort_sent_for_thinking(model, provider, SUMMARIZED_THINKING)
|
||||
sent: Final = _reasoning_effort_sent_for_thinking(model, provider, SUMMARIZED_THINKING)
|
||||
|
||||
assert sent == {"effort": "high", "summary": "auto"}
|
||||
|
||||
|
|
@ -122,7 +201,7 @@ class TestTheSummaryWrappingOnlyRidesTheResponsesBridge:
|
|||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true")
|
||||
|
||||
sent = _reasoning_effort_sent_for_thinking("azure/responses/gpt-5-mini", "azure", PLAIN_THINKING)
|
||||
sent: Final = _reasoning_effort_sent_for_thinking("azure/responses/gpt-5-mini", "azure", PLAIN_THINKING)
|
||||
|
||||
assert sent == {"effort": "high", "summary": "detailed"}
|
||||
|
||||
|
|
@ -139,21 +218,21 @@ class TestTheSummaryWrappingOnlyRidesTheResponsesBridge:
|
|||
"""``completion()`` keeps a gpt-5.5 deployment with function tools on Foundry's chat route when
|
||||
its ``api_base`` is a Foundry OpenAI host, and bridges it to Responses when the base makes it an
|
||||
Azure OpenAI deployment. The adapter has to read the same ``api_base`` to land on the same call."""
|
||||
sent = _reasoning_effort_sent_for_thinking(
|
||||
sent: Final = _reasoning_effort_sent_for_thinking(
|
||||
"azure_ai/gpt-5.5", "azure_ai", SUMMARIZED_THINKING, tools=[MULTIPLY_TOOL], api_base=api_base
|
||||
)
|
||||
|
||||
assert sent == expected
|
||||
|
||||
def test_a_provider_resolved_from_the_api_base_gets_the_plain_tier(self, local_model_cost_map: None) -> None:
|
||||
sent = _reasoning_effort_sent_for_thinking(
|
||||
sent: Final = _reasoning_effort_sent_for_thinking(
|
||||
"kimi-k3", None, SUMMARIZED_THINKING, api_base="https://api.together.xyz/v1"
|
||||
)
|
||||
|
||||
assert sent == "high"
|
||||
|
||||
def test_a_chained_gateway_keeps_the_dict_for_its_own_bridge(self, local_model_cost_map: None) -> None:
|
||||
sent = _reasoning_effort_sent_for_thinking("litellm_proxy/gpt-5.4", "litellm_proxy", SUMMARIZED_THINKING)
|
||||
sent: Final = _reasoning_effort_sent_for_thinking("litellm_proxy/gpt-5.4", "litellm_proxy", SUMMARIZED_THINKING)
|
||||
|
||||
assert sent == {"effort": "high", "summary": "auto"}
|
||||
|
||||
|
|
@ -192,7 +271,7 @@ class TestTheNormalizedTierIsTheTierSent:
|
|||
def test_the_dict_form_normalizes_effort_and_keeps_its_siblings(
|
||||
self, local_model_cost_map, model, provider, expected
|
||||
):
|
||||
sent = _reasoning_effort_sent(model, provider, {"effort": "max", "summary": "detailed"})
|
||||
sent: Final = _reasoning_effort_sent(model, provider, {"effort": "max", "summary": "detailed"})
|
||||
|
||||
assert sent == {"effort": expected, "summary": "detailed"}
|
||||
|
||||
|
|
@ -215,8 +294,8 @@ class TestTheNormalizedTierIsTheTierSent:
|
|||
],
|
||||
)
|
||||
def test_an_azure_hosted_astra_deployment_drops_to_the_tier_it_accepts(
|
||||
self, local_model_cost_map, model, provider
|
||||
):
|
||||
self, local_model_cost_map: None, model: str, provider: str
|
||||
) -> None:
|
||||
"""The deployment answers ``max`` with a 400 naming ``none`` through ``xhigh``, so the rows
|
||||
say so and the adapter sends the tier below instead of the rejected one."""
|
||||
assert _reasoning_effort_sent(model, provider, "max") == "xhigh"
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -3109,7 +3109,6 @@ class TestToolTransformation:
|
|||
("bedrock/converse/global.anthropic.claude-sonnet-5", "bedrock_converse"),
|
||||
("anthropic.claude-sonnet-4-5-20250929-v1:0", "bedrock"),
|
||||
("claude-sonnet-5", "vertex_ai"),
|
||||
("gemini-3.1-pro-preview", "vertex_ai"),
|
||||
("moonshotai.kimi-k2-thinking", "bedrock_mantle"),
|
||||
],
|
||||
)
|
||||
|
|
@ -3117,8 +3116,8 @@ class TestToolTransformation:
|
|||
"""
|
||||
A Responses request carrying ``reasoning.summary`` must still reach a chat provider as a
|
||||
plain ``reasoning_effort`` string. ``summary`` is Responses-only, and forwarding the whole
|
||||
object turns reasoning off: Bedrock Converse and Vertex silently discard a non-string
|
||||
``reasoning_effort``, and Bedrock Mantle rejects the request outright.
|
||||
object turns reasoning off: Bedrock Converse and Claude on Vertex silently discard a
|
||||
non-string ``reasoning_effort``, and Bedrock Mantle rejects the request outright.
|
||||
"""
|
||||
responses_api_request = {"reasoning": {"effort": "medium", "summary": "auto"}}
|
||||
|
||||
|
|
@ -3131,6 +3130,42 @@ class TestToolTransformation:
|
|||
|
||||
assert result["reasoning_effort"] == "medium"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, custom_llm_provider",
|
||||
[("gemini-3.1-pro-preview", "vertex_ai"), ("gemini-2.5-flash", "gemini")],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"reasoning, sends_thoughts",
|
||||
[({"effort": "medium", "summary": "auto"}, True), ({"effort": "medium"}, False)],
|
||||
)
|
||||
def test_a_summary_ask_turns_gemini_thoughts_on_over_the_gateway_default(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
reasoning: dict[str, str],
|
||||
sends_thoughts: bool,
|
||||
) -> None:
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
|
||||
from litellm.utils import reasoning_effort_with_summary_alias
|
||||
|
||||
monkeypatch.setattr(litellm, "gemini_include_thoughts_default", False)
|
||||
bridged: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
model=model,
|
||||
input="hi",
|
||||
responses_api_request={"reasoning": reasoning},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
gemini: Final = VertexGeminiConfig()
|
||||
reasoning_effort: Final = reasoning_effort_with_summary_alias(
|
||||
bridged["reasoning_effort"], bridged, accepts_summary=gemini.accepts_reasoning_summary_in_reasoning_effort
|
||||
)
|
||||
mapped: Final = gemini.map_openai_params({"reasoning_effort": reasoning_effort}, {}, model, True)
|
||||
|
||||
assert mapped["thinkingConfig"]["includeThoughts"] is sends_thoughts
|
||||
assert bridged["reasoning_effort"] == "medium"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, custom_llm_provider",
|
||||
[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue