diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml new file mode 100644 index 00000000000..ca274324f2f --- /dev/null +++ b/.github/workflows/test-unit-caching-redis.yml @@ -0,0 +1,38 @@ +name: "Unit Tests: Caching (Redis)" + +# Uses cloud Redis credentials — only runs on trusted branches, not PRs. +# This prevents external PRs from accessing Redis credentials. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + caching-redis: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + # Redis-only tests that do NOT require provider API keys. + # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py, + # test_router_caching.py) are in Phase 3 integration workflows. + test-path: >- + tests/local_testing/test_dual_cache.py + tests/local_testing/test_redis_batch_optimizations.py + tests/local_testing/test_router_utils.py + workers: 2 + reruns: 2 + timeout-minutes: 20 + enable-redis: true + enable-postgres: false + secrets: + REDIS_HOST: ${{ secrets.REDIS_HOST }} + REDIS_PORT: ${{ secrets.REDIS_PORT }} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/docs/my-website/docs/completion/web_search.md b/docs/my-website/docs/completion/web_search.md index 1f5ba2dee4e..86a96778616 100644 --- a/docs/my-website/docs/completion/web_search.md +++ b/docs/my-website/docs/completion/web_search.md @@ -596,3 +596,87 @@ Expected Response + +## Web Search Cost Tracking + +LiteLLM tracks web search costs automatically based on provider-specific billing models. The cost is added on top of the standard token-based pricing. + +### How providers charge for web search + +| Provider | Billing Unit | How it works | +|----------|-------------|--------------| +| **Gemini 3.x** (3-flash, 3-pro, 3.1-*) | Per search query | Each internal search query is billed individually. One prompt may trigger multiple queries. | +| **Gemini 2.x** (2.0-flash, 2.5-flash, 2.5-pro) | Per grounded prompt | Flat fee per API call that uses grounding, regardless of how many queries are executed internally. | +| **OpenAI** (gpt-4o-search, gpt-5-search) | Per search context size | Cost varies by `search_context_size` (`low`, `medium`, `high`). | +| **Anthropic** (Claude with web search) | Per search request | Fixed cost per web search tool invocation. | +| **Perplexity** (sonar, sonar-pro) | Per search context size | Cost varies by `search_context_size`. | + +### Pricing configuration + +Web search costs are defined in `model_prices_and_context_window.json` using two fields: + +- **`search_context_cost_per_query`**: the cost per billable unit (per search context size tier). +- **`web_search_billing_unit`** *(on Gemini models)*: `"per_query"` (each search query is billed individually) or `"per_prompt"` (default — flat fee per API call that uses search). + +```json +{ + "gemini/gemini-3-flash-preview": { + "web_search_billing_unit": "per_query", + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + } + }, + "gemini/gemini-2.5-flash": { + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } + } +} +``` + +:::info +Models without `web_search_billing_unit` default to `"per_prompt"` — one flat charge per API call that uses web search, regardless of how many internal queries the model executes. +::: + +You can override these in your proxy config using `model_info`: + +```yaml +model_list: + - model_name: gemini-3-flash + litellm_params: + model: gemini/gemini-3-flash-preview + model_info: + web_search_billing_unit: per_query + search_context_cost_per_query: + search_context_size_low: 0.014 + search_context_size_medium: 0.014 + search_context_size_high: 0.014 +``` + +### How LiteLLM tracks search usage + +The number of web search requests is stored in `usage.prompt_tokens_details.web_search_requests`. LiteLLM extracts this from each provider's response: + +- **Gemini**: Extracted from `groundingMetadata.webSearchQueries` in the response. For Gemini 2.x, clamped to 1 (per-prompt billing). +- **OpenAI**: Reported directly in the usage metadata. +- **Anthropic**: Reported via `server_tool_use.web_search_requests`. +- **xAI**: Mapped from `num_sources_used` in the response. + +```python +response = litellm.completion( + model="gemini/gemini-3-flash-preview", + messages=[{"role": "user", "content": "Latest tech news?"}], + web_search_options={"search_context_size": "medium"}, +) + +# Check web search usage +print(response.usage.prompt_tokens_details.web_search_requests) # e.g., 3 + +# Get total cost (includes token cost + web search cost) +cost = litellm.completion_cost(completion_response=response) +print(f"Total cost: ${cost}") +``` diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index b7a8b6f9ad7..e984df82140 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -90,6 +90,9 @@ _FINISH_REASON_MAP: dict[str, OpenAIChatCompletionFinishReason] = { "IMAGE_PROHIBITED_CONTENT": "content_filter", "TOO_MANY_TOOL_CALLS": "stop", "MALFORMED_RESPONSE": "stop", + # Zhipu GLM + "network_error": "stop", + "sensitive": "content_filter", # Bedrock "guardrail_intervened": "content_filter", # OpenAI passthrough diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 5a19c224aa4..5a95d12f5b3 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4050,6 +4050,40 @@ def _deduplicate_bedrock_tool_content( return _deduplicate_bedrock_content_blocks(tool_content, "toolResult") +def _sort_bedrock_assistant_content_blocks( + blocks: List[BedrockContentBlock], +) -> List[BedrockContentBlock]: + """ + Sort assistant content blocks so that ``text`` blocks appear before + ``toolUse`` blocks. + + Bedrock requires all ``text`` blocks to precede any ``toolUse`` blocks + within an assistant message. When the Responses API converts + function_call items before message items, the resulting ``toolUse`` + blocks can end up before ``text`` blocks, causing Bedrock to reject + the request with a 400 error because the ``toolUse`` → ``toolResult`` + pairing is broken by the intervening ``text`` block. + + Sort order (stable): + 0 - reasoningContent + 1 - text / image / document / video / other non-tool blocks + 2 - toolUse + """ + + def _sort_key(block: BedrockContentBlock) -> int: + if "reasoningContent" in block: + return 0 + if "toolUse" in block: + return 2 + if "cachePoint" in block: + # cachePoint blocks are paired with their preceding toolUse block. + # Same key as toolUse so Python's stable sort keeps them together. + return 2 + return 1 + + return sorted(blocks, key=_sort_key) + + def _insert_assistant_continue_message( messages: List[BedrockMessageBlock], assistant_continue_message: Optional[ @@ -4643,6 +4677,9 @@ class BedrockConverseMessagesProcessor: assistant_content = _deduplicate_bedrock_content_blocks( assistant_content, "toolUse" ) + assistant_content = _sort_bedrock_assistant_content_blocks( + assistant_content + ) if assistant_content: contents.append( @@ -5008,6 +5045,7 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 assistant_content = _deduplicate_bedrock_content_blocks( assistant_content, "toolUse" ) + assistant_content = _sort_bedrock_assistant_content_blocks(assistant_content) if assistant_content: contents.append( diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index c808cfca457..fe7c62c3842 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -119,7 +119,8 @@ class ChunkProcessor: model = ChunkProcessor._get_model_from_chunks(chunks, first_chunk_model) system_fingerprint = chunk.get("system_fingerprint", None) - role = chunk["choices"][0]["delta"]["role"] + first_chunk_with_choices = next((c for c in chunks if c.get("choices")), chunk) + role = first_chunk_with_choices["choices"][0]["delta"]["role"] finish_reason = "stop" for chunk in chunks: if "choices" in chunk and len(chunk["choices"]) > 0: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index e350b547be9..e281b172685 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -831,6 +831,11 @@ class CustomStreamWrapper: "annotations" in model_response.choices[0].delta and model_response.choices[0].delta.annotations is not None ) + or ( + not self.sent_first_chunk + and hasattr(model_response.choices[0].delta, "role") + and model_response.choices[0].delta.role is not None + ) or ( getattr(model_response.choices[0].delta, "reasoning_items", None) is not None @@ -1564,6 +1569,7 @@ class CustomStreamWrapper: self.stream_options is not None and self.stream_options["include_usage"] is True ): + model_response.choices = [] return model_response return ## CHECK FOR TOOL USE @@ -1863,11 +1869,14 @@ class CustomStreamWrapper: response, cache_hit, ) # log response - choice = response.choices[0] - if isinstance(choice, StreamingChoices): - self.response_uptil_now += choice.delta.get("content", "") or "" - else: - self.response_uptil_now += "" + if response.choices: + choice = response.choices[0] + if isinstance(choice, StreamingChoices): + self.response_uptil_now += ( + choice.delta.get("content", "") or "" + ) + else: + self.response_uptil_now += "" self.rules.post_call_rules( input=self.response_uptil_now, model=self.model ) @@ -1875,7 +1884,7 @@ class CustomStreamWrapper: self.chunks.append(response) # Add mcp_list_tools to first chunk if present - if not self.sent_first_chunk: + if not self.sent_first_chunk and response.choices: response = self._add_mcp_list_tools_to_first_chunk(response) self.sent_first_chunk = True @@ -2043,16 +2052,19 @@ class CustomStreamWrapper: completion_start_time=datetime.datetime.now() ) - choice = processed_chunk.choices[0] - if isinstance(choice, StreamingChoices): - self.response_uptil_now += choice.delta.get("content", "") or "" - else: - self.response_uptil_now += "" + if processed_chunk.choices: + choice = processed_chunk.choices[0] + if isinstance(choice, StreamingChoices): + self.response_uptil_now += ( + choice.delta.get("content", "") or "" + ) + else: + self.response_uptil_now += "" self.rules.post_call_rules( input=self.response_uptil_now, model=self.model ) # Add mcp_list_tools to first chunk if present - if not self.sent_first_chunk: + if not self.sent_first_chunk and processed_chunk.choices: processed_chunk = self._add_mcp_list_tools_to_first_chunk( processed_chunk ) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index 799e8ab9a0a..c65dfb22730 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -338,8 +338,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): "index": max(self.current_content_block_index - 1, 0), } ) - - # 2. Start new content block self.chunk_queue.append( { "type": "content_block_start", @@ -361,8 +359,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Reset state for new block self.sent_content_block_finish = False - - # Return the first queued item return self.chunk_queue.popleft() if ( diff --git a/litellm/llms/gemini/cost_calculator.py b/litellm/llms/gemini/cost_calculator.py index 45850e0d668..cd536b8bd3e 100644 --- a/litellm/llms/gemini/cost_calculator.py +++ b/litellm/llms/gemini/cost_calculator.py @@ -30,15 +30,24 @@ def cost_per_token( def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> float: """ - Calculates the cost per web search request for a given model, prompt tokens, and completion tokens. + Calculates the cost of web search (grounding with Google Search). + + Billing mode is determined by ``web_search_billing_unit`` in model_info: + - ``"per_query"``: charged per individual search query (Gemini 3.x). + - ``"per_prompt"`` (default): charged per grounded prompt (Gemini 2.x), + regardless of how many queries were executed internally. + + Reads the per-request cost from ``search_context_cost_per_query`` in + ``model_info`` when available, falling back to $0.035 for models not + yet updated in the pricing JSON. """ from litellm.types.utils import PromptTokensDetailsWrapper - # cost per web search request - cost_per_web_search_request = 35e-3 + _DEFAULT_COST = 35e-3 + search_costs = model_info.get("search_context_cost_per_query") or {} + _cost = search_costs.get("search_context_size_medium", _DEFAULT_COST) number_of_web_search_requests = 0 - # Get number of web search requests if ( usage is not None and usage.prompt_tokens_details is not None @@ -47,10 +56,10 @@ def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> floa and usage.prompt_tokens_details.web_search_requests is not None ): number_of_web_search_requests = usage.prompt_tokens_details.web_search_requests - else: - number_of_web_search_requests = 0 - # Calculate total cost - total_cost = cost_per_web_search_request * number_of_web_search_requests + # per_prompt billing: clamp to 1 (flat fee per grounded API call) + billing_mode = model_info.get("web_search_billing_unit", "per_prompt") + if number_of_web_search_requests > 0 and billing_mode == "per_prompt": + number_of_web_search_requests = 1 - return total_cost + return _cost * number_of_web_search_requests diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index cafb745862d..87c502032cc 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -35,6 +35,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): def supports_native_file_search(self) -> bool: return True + @staticmethod + def _is_gpt_5_model(model: str) -> bool: + """Return True only for actual OpenAI GPT-5 models. + + Excludes pass-through models from other providers that happen to + reference gpt-5 in their name (e.g. perplexity/openai/gpt-5.2). + """ + parts = model.split("/") + if len(parts) > 1 and parts[0] not in ("openai",): + return False + return "gpt-5" in model and "gpt-5-chat" not in model + + @staticmethod + def _supports_reasoning_effort_none(model: str) -> bool: + """Return True if the model supports reasoning.effort='none'.""" + from litellm.utils import _supports_factory + + return _supports_factory( + model=model, + custom_llm_provider=None, + key="supports_none_reasoning_effort", + ) + def get_supported_openai_params(self, model: str) -> list: """ All OpenAI Responses API params are supported @@ -60,8 +83,39 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): model: str, drop_params: bool, ) -> Dict: - """No mapping applied since inputs are in OpenAI spec already""" - return dict(response_api_optional_params) + """No mapping applied since inputs are in OpenAI spec already. + + GPT-5 models have restrictions on temperature (only temperature=1 + is accepted unless reasoning_effort='none' on models that support it). + Apply the same validation used by the chat completions path. + """ + params = dict(response_api_optional_params) + + if self._is_gpt_5_model(model=model): + temperature = params.get("temperature") + if temperature is not None and temperature != 1: + reasoning = params.get("reasoning") or {} + effort = ( + reasoning.get("effort") if isinstance(reasoning, dict) else None + ) + supports_none = self._supports_reasoning_effort_none(model=model) + if supports_none and (effort == "none" or effort is None): + pass # flexible temperature allowed + elif drop_params or litellm.drop_params: + params.pop("temperature", None) + else: + raise litellm.UnsupportedParamsError( + message=( + "gpt-5 models don't support temperature={}. " + "Only temperature=1 is supported. " + "For models like gpt-5.1/5.4, temperature is supported " + "when reasoning.effort='none' (or not specified). " + "To drop unsupported params set `litellm.drop_params = True`" + ).format(temperature), + status_code=400, + ) + + return params def transform_responses_api_request( self, diff --git a/litellm/llms/vertex_ai/gemini/cost_calculator.py b/litellm/llms/vertex_ai/gemini/cost_calculator.py index 23977bc9170..69a29c16afd 100644 --- a/litellm/llms/vertex_ai/gemini/cost_calculator.py +++ b/litellm/llms/vertex_ai/gemini/cost_calculator.py @@ -1,7 +1,8 @@ """ Cost calculator for Vertex AI Gemini. -Used because there are differences in how Google AI Studio and Vertex AI Gemini handle web search requests. +Delegates to the shared Gemini cost calculator which reads pricing and +billing unit from model_info. """ from typing import TYPE_CHECKING @@ -14,32 +15,14 @@ def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> floa """ Calculate the cost of a web search request for Vertex AI Gemini. - Vertex AI charges $35/1000 prompts, independent of the number of web search requests. + Billing differs by ``web_search_billing_unit`` in ``model_info``: + - ``"per_query"``: charged per individual search query (Gemini 3.x). + - ``"per_prompt"`` (default): charged per grounded prompt (Gemini 2.x). - For a single call, this is $35e-3 USD. - - Args: - usage: The usage object for the web search request. - model_info: The model info for the web search request. - - Returns: - The cost of the web search request. + Delegates to the shared Gemini cost calculator. """ - from litellm.types.utils import PromptTokensDetailsWrapper + from litellm.llms.gemini.cost_calculator import ( + cost_per_web_search_request as _gemini_cost, + ) - # check if usage object has web search requests - cost_per_llm_call_with_web_search = 35e-3 - - makes_web_search_request = False - if ( - usage is not None - and usage.prompt_tokens_details is not None - and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper) - ): - makes_web_search_request = True - - # Calculate total cost - if makes_web_search_request: - return cost_per_llm_call_with_web_search - else: - return 0.0 + return _gemini_cost(usage=usage, model_info=model_info) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index cd27b4c362a..cc81218a13b 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2462,6 +2462,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): usage = VertexGeminiConfig._calculate_usage( completion_response=completion_response ) + + web_search_requests = VertexGeminiConfig._calculate_web_search_requests( + grounding_metadata + ) + if web_search_requests is not None: + cast( + PromptTokensDetailsWrapper, usage.prompt_tokens_details + ).web_search_requests = web_search_requests + setattr(model_response, "usage", usage) ## ADD METADATA TO RESPONSE ## diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 389a3a85f56..34fc95e0af7 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -141,6 +141,19 @@ def _is_multimodal_input(input: EmbeddingInput) -> bool: return False +_SUPPORTED_EMBED_PARAMS = {"outputDimensionality", "taskType", "title"} + + +def _filter_embed_params(optional_params: dict) -> dict: + """Map and filter optional_params to only include Gemini embedding fields.""" + gemini_params = optional_params.copy() + if "dimensions" in gemini_params: + gemini_params["outputDimensionality"] = gemini_params.pop("dimensions") + if "task_type" in gemini_params: + gemini_params["taskType"] = gemini_params.pop("task_type") + return {k: v for k, v in gemini_params.items() if k in _SUPPORTED_EMBED_PARAMS} + + def transform_openai_input_gemini_content( input: EmbeddingInput, model: str, optional_params: dict ) -> VertexAIBatchEmbeddingsRequestBody: @@ -149,11 +162,7 @@ def transform_openai_input_gemini_content( """ gemini_model_name = "models/{}".format(model) - gemini_params = optional_params.copy() - if "dimensions" in gemini_params: - gemini_params["outputDimensionality"] = gemini_params.pop("dimensions") - if "task_type" in gemini_params: - gemini_params["taskType"] = gemini_params.pop("task_type") + gemini_params = _filter_embed_params(optional_params) requests: List[EmbedContentRequest] = [] if isinstance(input, str): @@ -195,11 +204,7 @@ def transform_openai_input_gemini_embed_content( """ resolved_files = resolved_files or {} - gemini_params = optional_params.copy() - if "dimensions" in gemini_params: - gemini_params["outputDimensionality"] = gemini_params.pop("dimensions") - if "task_type" in gemini_params: - gemini_params["taskType"] = gemini_params.pop("task_type") + gemini_params = _filter_embed_params(optional_params) input_list = [input] if isinstance(input, str) else input parts: List[PartType] = [] diff --git a/litellm/main.py b/litellm/main.py index a93aed51b20..daa0fb063aa 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7397,8 +7397,9 @@ def stream_chunk_builder( # noqa: PLR0915 if len(chunks) == 0: return None ## Route to the text completion logic - if isinstance( - chunks[0]["choices"][0], litellm.utils.TextChoices + first_chunk_with_choices = next((c for c in chunks if c["choices"]), None) + if first_chunk_with_choices is not None and isinstance( + first_chunk_with_choices["choices"][0], litellm.utils.TextChoices ): # route to the text completion logic return stream_chunk_builder_text_completion( chunks=chunks, messages=messages diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 98811e8a01d..6d7a8a87898 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13945,7 +13945,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.0-flash-001": { "cache_read_input_token_cost": 3.75e-08, @@ -13983,7 +13988,12 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.0-flash-lite": { "cache_read_input_token_cost": 1.875e-08, @@ -14019,7 +14029,12 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.0-flash-lite-001": { "cache_read_input_token_cost": 1.875e-08, @@ -14055,7 +14070,12 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-flash": { "cache_read_input_token_cost": 3e-08, @@ -14101,6 +14121,11 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini-2.5-flash-image": { @@ -14186,6 +14211,12 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", "supports_service_tier": true }, "gemini-3.1-flash-image-preview": { @@ -14218,7 +14249,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -14270,6 +14307,12 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", "supports_service_tier": true }, "deep-research-pro-preview-12-2025": { @@ -14350,6 +14393,11 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini-2.5-flash-lite-preview-09-2025": { @@ -14395,7 +14443,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -14440,7 +14493,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -14484,7 +14542,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -14530,7 +14593,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 8000000 + "tpm": 8000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-flash-lite-preview-06-17": { "deprecation_date": "2025-11-18", @@ -14576,7 +14644,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -14622,6 +14695,11 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini-3-pro-preview": { @@ -14679,7 +14757,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -14737,7 +14821,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -14788,7 +14878,13 @@ "supports_vision": true, "supports_web_search": true, "supports_url_context": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -14844,7 +14940,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, @@ -14893,7 +14995,13 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -14951,7 +15059,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -15009,7 +15123,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, @@ -15045,7 +15165,12 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-robotics-er-1.5-preview": { "cache_read_input_token_cost": 0, @@ -15120,7 +15245,12 @@ "supports_vision": true, "supports_web_search": true, "tpm": 250000, - "rpm": 10 + "rpm": 10, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, @@ -15187,17 +15317,14 @@ "uses_embed_content": true }, "vertex_ai/gemini-embedding-2-preview": { - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, - "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_token": 1.5e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal", "supports_multimodal": true, "uses_embed_content": true }, @@ -15327,7 +15454,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 10000000 + "tpm": 10000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.0-flash-001": { "cache_read_input_token_cost": 2.5e-08, @@ -15366,7 +15498,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "tpm": 10000000 + "tpm": 10000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.0-flash-lite": { "cache_read_input_token_cost": 1.875e-08, @@ -15403,7 +15540,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "tpm": 4000000 + "tpm": 4000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-flash": { "cache_read_input_token_cost": 3e-08, @@ -15451,6 +15593,11 @@ "supports_vision": true, "supports_web_search": true, "tpm": 8000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini/gemini-2.5-flash-image": { @@ -15502,6 +15649,11 @@ "supports_vision": true, "supports_web_search": true, "tpm": 8000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini/gemini-3-pro-image-preview": { @@ -15539,6 +15691,12 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", "supports_service_tier": true }, "gemini/gemini-3.1-flash-image-preview": { @@ -15575,7 +15733,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, @@ -15611,7 +15775,12 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-flash-lite": { "cache_read_input_token_cost": 1e-08, @@ -15659,6 +15828,11 @@ "supports_vision": true, "supports_web_search": true, "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { @@ -15706,7 +15880,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -15753,7 +15932,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, @@ -15800,7 +15984,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-flash-lite-latest": { "cache_read_input_token_cost": 2.5e-08, @@ -15847,7 +16036,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-flash-lite-preview-06-17": { "deprecation_date": "2025-11-18", @@ -15895,7 +16089,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 3e-07, @@ -15958,7 +16157,12 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "tpm": 800000 + "tpm": 800000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, @@ -16046,7 +16250,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -16100,6 +16310,12 @@ "supports_web_search": true, "supports_native_streaming": true, "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", "supports_service_tier": true }, "gemini/gemini-3-flash-preview": { @@ -16153,7 +16369,13 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -16211,7 +16433,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -16269,7 +16497,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, @@ -16320,7 +16554,13 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, @@ -16357,7 +16597,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "tpm": 10000000 + "tpm": 10000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-exp-1114": { "input_cost_per_token": 0, @@ -32170,7 +32415,13 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "vertex_ai/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, @@ -38178,7 +38429,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "tpm": 4000000 + "tpm": 4000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 1e-06, @@ -38451,7 +38707,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 8000000 + "tpm": 8000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-flash-lite-latest": { "cache_read_input_token_cost": 1e-08, @@ -38498,7 +38759,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-pro-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -38544,7 +38810,12 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "tpm": 800000 + "tpm": 800000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-pro-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -38590,7 +38861,12 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "tpm": 800000 + "tpm": 800000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-exp-1206": { "cache_read_input_token_cost": 3e-08, @@ -38637,7 +38913,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 8000000 + "tpm": 8000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "vertex_ai/claude-sonnet-4-6@default": { "cache_creation_input_token_cost": 3.75e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a6b66cdb75a..8b2242f9187 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13959,7 +13959,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.0-flash-001": { "cache_read_input_token_cost": 3.75e-08, @@ -13997,7 +14002,12 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.0-flash-lite": { "cache_read_input_token_cost": 1.875e-08, @@ -14033,7 +14043,12 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.0-flash-lite-001": { "cache_read_input_token_cost": 1.875e-08, @@ -14069,7 +14084,12 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-flash": { "cache_read_input_token_cost": 3e-08, @@ -14115,6 +14135,11 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini-2.5-flash-image": { @@ -14200,6 +14225,12 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", "supports_service_tier": true }, "gemini-3.1-flash-image-preview": { @@ -14232,7 +14263,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -14284,6 +14321,12 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", "supports_service_tier": true }, "deep-research-pro-preview-12-2025": { @@ -14364,6 +14407,11 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini-2.5-flash-lite-preview-09-2025": { @@ -14409,7 +14457,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -14454,7 +14507,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -14498,7 +14556,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -14544,7 +14607,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 8000000 + "tpm": 8000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-flash-lite-preview-06-17": { "deprecation_date": "2025-11-18", @@ -14590,7 +14658,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -14636,6 +14709,11 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini-3-pro-preview": { @@ -14693,7 +14771,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -14751,7 +14835,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -14802,7 +14892,13 @@ "supports_vision": true, "supports_web_search": true, "supports_url_context": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -14858,7 +14954,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, @@ -14907,7 +15009,13 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -14965,7 +15073,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -15023,7 +15137,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, @@ -15059,7 +15179,12 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-robotics-er-1.5-preview": { "cache_read_input_token_cost": 0, @@ -15134,7 +15259,12 @@ "supports_vision": true, "supports_web_search": true, "tpm": 250000, - "rpm": 10 + "rpm": 10, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, @@ -15201,17 +15331,14 @@ "uses_embed_content": true }, "vertex_ai/gemini-embedding-2-preview": { - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, - "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_token": 1.5e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal", "supports_multimodal": true, "uses_embed_content": true }, @@ -15341,7 +15468,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 10000000 + "tpm": 10000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.0-flash-001": { "cache_read_input_token_cost": 2.5e-08, @@ -15380,7 +15512,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "tpm": 10000000 + "tpm": 10000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.0-flash-lite": { "cache_read_input_token_cost": 1.875e-08, @@ -15417,7 +15554,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "tpm": 4000000 + "tpm": 4000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-flash": { "cache_read_input_token_cost": 3e-08, @@ -15465,6 +15607,11 @@ "supports_vision": true, "supports_web_search": true, "tpm": 8000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini/gemini-2.5-flash-image": { @@ -15516,6 +15663,11 @@ "supports_vision": true, "supports_web_search": true, "tpm": 8000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini/gemini-3-pro-image-preview": { @@ -15553,6 +15705,12 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", "supports_service_tier": true }, "gemini/gemini-3.1-flash-image-preview": { @@ -15589,7 +15747,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, @@ -15625,7 +15789,12 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-flash-lite": { "cache_read_input_token_cost": 1e-08, @@ -15673,6 +15842,11 @@ "supports_vision": true, "supports_web_search": true, "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, "supports_service_tier": true }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { @@ -15720,7 +15894,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -15767,7 +15946,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, @@ -15814,7 +15998,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-flash-lite-latest": { "cache_read_input_token_cost": 2.5e-08, @@ -15861,7 +16050,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-flash-lite-preview-06-17": { "deprecation_date": "2025-11-18", @@ -15909,7 +16103,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 3e-07, @@ -15972,7 +16171,12 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "tpm": 800000 + "tpm": 800000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, @@ -16060,7 +16264,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -16114,6 +16324,12 @@ "supports_web_search": true, "supports_native_streaming": true, "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", "supports_service_tier": true }, "gemini/gemini-3-flash-preview": { @@ -16167,7 +16383,13 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -16225,7 +16447,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-pro-preview-customtools": { "cache_read_input_token_cost": 2e-07, @@ -16283,7 +16511,13 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, @@ -16334,7 +16568,13 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, - "supports_service_tier": true + "supports_service_tier": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, @@ -16371,7 +16611,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "tpm": 10000000 + "tpm": 10000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-exp-1114": { "input_cost_per_token": 0, @@ -32206,7 +32451,13 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" }, "vertex_ai/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, @@ -38241,7 +38492,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "tpm": 4000000 + "tpm": 4000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 1e-06, @@ -38514,7 +38770,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 8000000 + "tpm": 8000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-flash-lite-latest": { "cache_read_input_token_cost": 1e-08, @@ -38561,7 +38822,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 250000 + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-pro-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -38607,7 +38873,12 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "tpm": 800000 + "tpm": 800000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini/gemini-pro-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -38653,7 +38924,12 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "tpm": 800000 + "tpm": 800000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "gemini-exp-1206": { "cache_read_input_token_cost": 3e-08, @@ -38700,7 +38976,12 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "tpm": 8000000 + "tpm": 8000000, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + } }, "vertex_ai/claude-sonnet-4-6@default": { "cache_creation_input_token_cost": 3.75e-06, diff --git a/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py b/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py index dba5c0e902e..54ea41a6450 100644 --- a/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py +++ b/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py @@ -19,6 +19,7 @@ import pytest import litellm from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( + _filter_embed_params, _is_multimodal_input, _parse_data_url, process_embed_content_response, @@ -573,6 +574,53 @@ def test_vertex_ai_text_only_embedding_uses_embed_content(): assert len(response.data) == 1 +# --------------------------------------------------------------------------- +# Unsupported params filtering tests (#24293) +# --------------------------------------------------------------------------- + + +def test_filter_embed_params_drops_unsupported(): + """Unsupported params like max_tokens should be filtered out.""" + result = _filter_embed_params({"dimensions": 768, "max_tokens": 256, "temperature": 0.5}) + assert result == {"outputDimensionality": 768} + + +def test_filter_embed_params_keeps_supported(): + """All supported Gemini embedding params should pass through.""" + result = _filter_embed_params({ + "dimensions": 768, + "task_type": "RETRIEVAL_DOCUMENT", + "title": "My doc", + }) + assert result == { + "outputDimensionality": 768, + "taskType": "RETRIEVAL_DOCUMENT", + "title": "My doc", + } + + +def test_batch_embed_content_drops_max_tokens(): + """max_tokens in optional_params should not appear in the batch request.""" + result = transform_openai_input_gemini_content( + input="test text", + model="text-embedding-004", + optional_params={"max_tokens": 256}, + ) + for request in result["requests"]: + assert "max_tokens" not in request + + +def test_embed_content_drops_max_tokens(): + """max_tokens in optional_params should not appear in the embedContent request.""" + result = transform_openai_input_gemini_embed_content( + input=["test text"], + model="gemini-embedding-001", + optional_params={"max_tokens": 256}, + resolved_files=None, + ) + assert "max_tokens" not in result + + def test_batch_embeddings_response_has_correct_indices_and_order(): """Test that process_response assigns sequential indices and preserves order.""" response_json = { diff --git a/tests/litellm_core_utils/test_bedrock_converse_dedup_factory.py b/tests/litellm_core_utils/test_bedrock_converse_dedup_factory.py index 6cffa4d1d55..c32917efe87 100644 --- a/tests/litellm_core_utils/test_bedrock_converse_dedup_factory.py +++ b/tests/litellm_core_utils/test_bedrock_converse_dedup_factory.py @@ -8,6 +8,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, _deduplicate_bedrock_content_blocks, _deduplicate_bedrock_tool_content, + _sort_bedrock_assistant_content_blocks, BedrockConverseMessagesProcessor, ) @@ -450,3 +451,133 @@ def test_bedrock_converse_filters_empty_list_content(): assert len(text_blocks) == 2 assert text_blocks[0]["text"] == "Hello" assert text_blocks[1]["text"] == "World" + + +# --------------------------------------------------------------------------- +# Content block ordering tests (text before toolUse) +# --------------------------------------------------------------------------- + + +def _make_tooluse_before_text_messages(): + """Return messages where the assistant message has a tool_call followed by + a separate assistant message with text content. When merged, the toolUse + block would end up before the text block without sorting.""" + return [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "tooluse_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + } + ], + }, + { + "role": "assistant", + "content": "Let me check the weather for you.", + }, + { + "role": "tool", + "tool_call_id": "tooluse_abc123", + "content": '{"temp": 22}', + }, + ] + + +def test_sort_bedrock_assistant_content_blocks_text_before_tooluse(): + """Direct unit test: text blocks should come before toolUse blocks.""" + blocks = [ + {"toolUse": {"toolUseId": "id_1", "name": "fn_a", "input": {}}}, + {"text": "thinking..."}, + ] + + result = _sort_bedrock_assistant_content_blocks(blocks) + + assert len(result) == 2 + assert "text" in result[0] + assert "toolUse" in result[1] + + +def test_sort_bedrock_assistant_content_blocks_reasoning_first(): + """reasoningContent blocks should come before text and toolUse.""" + blocks = [ + {"toolUse": {"toolUseId": "id_1", "name": "fn_a", "input": {}}}, + {"text": "thinking..."}, + {"reasoningContent": {"reasoningText": {"text": "reasoning"}}}, + ] + + result = _sort_bedrock_assistant_content_blocks(blocks) + + assert "reasoningContent" in result[0] + assert "text" in result[1] + assert "toolUse" in result[2] + + +def test_sort_bedrock_assistant_content_blocks_preserves_order_when_correct(): + """If blocks are already in the correct order, sorting should not change them.""" + blocks = [ + {"text": "hello"}, + {"toolUse": {"toolUseId": "id_1", "name": "fn_a", "input": {}}}, + {"toolUse": {"toolUseId": "id_2", "name": "fn_b", "input": {}}}, + ] + + result = _sort_bedrock_assistant_content_blocks(blocks) + + assert result == blocks + + +def test_bedrock_converse_sorts_text_before_tooluse_sync(): + """Verify the sync path sorts text blocks before toolUse blocks in + assistant messages.""" + messages = _make_tooluse_before_text_messages() + result = _bedrock_converse_messages_pt(messages, MODEL, PROVIDER) + + assistant_msgs = [msg for msg in result if msg["role"] == "assistant"] + assert len(assistant_msgs) == 1 + + content = assistant_msgs[0]["content"] + text_indices = [i for i, b in enumerate(content) if "text" in b] + tool_indices = [i for i, b in enumerate(content) if "toolUse" in b] + + # All text blocks must come before all toolUse blocks + assert max(text_indices) < min(tool_indices), ( + f"text blocks at {text_indices} should all precede toolUse blocks at {tool_indices}" + ) + + +@pytest.mark.asyncio +async def test_bedrock_converse_sorts_text_before_tooluse_async(): + """Verify the async path sorts text blocks before toolUse blocks in + assistant messages.""" + messages = _make_tooluse_before_text_messages() + result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages, MODEL, PROVIDER + ) + + assistant_msgs = [msg for msg in result if msg["role"] == "assistant"] + assert len(assistant_msgs) == 1 + + content = assistant_msgs[0]["content"] + text_indices = [i for i, b in enumerate(content) if "text" in b] + tool_indices = [i for i, b in enumerate(content) if "toolUse" in b] + + assert max(text_indices) < min(tool_indices), ( + f"text blocks at {text_indices} should all precede toolUse blocks at {tool_indices}" + ) + + +@pytest.mark.asyncio +async def test_bedrock_converse_content_ordering_sync_async_parity(): + """Sync and async paths should produce identical content block ordering.""" + messages = _make_tooluse_before_text_messages() + sync_result = _bedrock_converse_messages_pt(messages, MODEL, PROVIDER) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages, MODEL, PROVIDER + ) + assert sync_result == async_result diff --git a/tests/llm_translation/test_gpt4o_audio.py b/tests/llm_translation/test_gpt4o_audio.py index b322555bb85..4b70256335e 100644 --- a/tests/llm_translation/test_gpt4o_audio.py +++ b/tests/llm_translation/test_gpt4o_audio.py @@ -34,8 +34,10 @@ async def check_streaming_response(completion): _audio_id = None async for chunk in completion: print(chunk) + if len(chunk.choices) == 0: + continue _choice: StreamingChoices = chunk.choices[0] - if _choice.delta.audio is not None: + if _choice.delta is not None and _choice.delta.audio is not None: if _choice.delta.audio.get("data") is not None: _audio_bytes = _choice.delta.audio["data"] if _choice.delta.audio.get("transcript") is not None: diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index d6370bfd049..6341fa78006 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -1759,8 +1759,14 @@ def test_completion_logprobs_stream(): for chunk in response: # check if atleast one chunk has log probs print(chunk) + if len(chunk.choices) == 0: + continue print(f"chunk.choices[0]: {chunk.choices[0]}") - if "logprobs" in chunk.choices[0]: + if ( + "logprobs" in chunk.choices[0] + and chunk.choices[0].logprobs is not None + and len(chunk.choices[0].logprobs.content) > 0 + ): # assert we got a valid logprob in the choices assert len(chunk.choices[0].logprobs.content[0].top_logprobs) == 3 found_logprob = True diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index ecac2cfe40e..bc3c34b1a5d 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -831,23 +831,29 @@ def test_completion_mistral_api_mistral_large_function_call_with_streaming(): tool_choice="auto", stream=True, ) - idx = 0 + saw_function_call_chunk = False for chunk in response: print(f"chunk in response: {chunk}") assert chunk._hidden_params["custom_llm_provider"] == "mistral" - if idx == 0: - assert ( - chunk.choices[0].delta.tool_calls[0].function.arguments is not None - ) - assert isinstance( - chunk.choices[0].delta.tool_calls[0].function.arguments, str - ) - validate_first_streaming_function_calling_chunk(chunk=chunk) - elif idx == 1 and chunk.choices[0].finish_reason is None: - validate_second_streaming_function_calling_chunk(chunk=chunk) - elif chunk.choices[0].finish_reason is not None: # last chunk + if len(chunk.choices) == 0: + continue + if chunk.choices[0].finish_reason is not None: # last chunk validate_final_streaming_function_calling_chunk(chunk=chunk) - idx += 1 + break + tool_calls = chunk.choices[0].delta.tool_calls + if tool_calls is None: + continue + assert tool_calls[0].function.arguments is not None + assert isinstance(tool_calls[0].function.arguments, str) + if not saw_function_call_chunk: + if chunk.choices[0].delta.role is not None: + validate_first_streaming_function_calling_chunk(chunk=chunk) + else: + validate_second_streaming_function_calling_chunk(chunk=chunk) + saw_function_call_chunk = True + else: + validate_second_streaming_function_calling_chunk(chunk=chunk) + assert saw_function_call_chunk except litellm.RateLimitError: pass except Exception as e: diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py index aa5ce5fa6a4..b67ea91bb0b 100644 --- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py @@ -135,6 +135,14 @@ class TestMapFinishReasonBedrock: assert map_finish_reason("guardrail_intervened") == "content_filter" +class TestMapFinishReasonZhipu: + def test_network_error(self): + assert map_finish_reason("network_error") == "stop" + + def test_sensitive(self): + assert map_finish_reason("sensitive") == "content_filter" + + class TestMapFinishReasonOpenAIPassthrough: @pytest.mark.parametrize( "reason", ["stop", "length", "tool_calls", "function_call", "content_filter"] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 28cf087c7e7..d6281703a0a 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1879,6 +1879,150 @@ async def test_custom_stream_wrapper_anext_exhaustion_raises_stop_async_iteratio pytest.fail(f"PEP 479 regression: StopIteration leaked as RuntimeError: {e}") +# Azure streaming chunks that reproduce issue #24221: +# Azure sends an initial chunk with prompt_filter_results and choices=[], +# then a chunk with role='assistant' and content='', then content chunks. +# With stream_options.include_usage=True, the empty-choices chunk was +# forwarded with an inflated default choice, consuming the sent_first_chunk +# flag and causing strip_role_from_delta to strip the role from the real +# first chunk. +_AZURE_CHUNKS_WITH_PROMPT_FILTER = [ + # Chunk 1: prompt_filter_results, no choices (Azure-specific) + ModelResponseStream( + id="chatcmpl-abc123", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[], + usage=None, + ), + # Chunk 2: first real chunk with role='assistant' and empty content + ModelResponseStream( + id="chatcmpl-abc123", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="", role="assistant"), + ) + ], + usage=None, + ), + # Chunk 3: content + ModelResponseStream( + id="chatcmpl-abc123", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="Hello!"), + ) + ], + usage=None, + ), + # Chunk 4: finish_reason + ModelResponseStream( + id="chatcmpl-abc123", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ], + usage=None, + ), + # Chunk 5: final usage chunk, no choices + ModelResponseStream( + id="chatcmpl-abc123", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[], + usage=Usage( + completion_tokens=10, + prompt_tokens=20, + total_tokens=30, + ), + ), +] + + +@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) +@pytest.mark.asyncio +async def test_azure_streaming_role_preserved_with_include_usage(sync_mode: bool): + """ + Regression test for https://github.com/BerriAI/litellm/issues/24221 + + Azure sends an initial chunk with choices=[] (prompt_filter_results) + before the first content chunk. With stream_options.include_usage=True, + this chunk was forwarded with an inflated default choice, which: + 1. Consumed the sent_first_chunk flag + 2. Caused strip_role_from_delta to strip role from the real first chunk + + The fix ensures: + - Chunks with choices=[] are forwarded faithfully (no inflated choices) + - sent_first_chunk is only marked for chunks with real choices + - Chunks with role in delta are not discarded as empty + """ + completion_stream = ModelResponseListIterator( + model_responses=_AZURE_CHUNKS_WITH_PROMPT_FILTER + ) + + response = CustomStreamWrapper( + completion_stream=completion_stream, + model="azure/gpt-5-nano", + custom_llm_provider="azure", + logging_obj=Logging( + model="azure/gpt-5-nano", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="12345", + function_id="1245", + ), + stream_options={"include_usage": True}, + ) + + chunks = [] + if sync_mode: + for chunk in response: + chunks.append(chunk) + else: + async for chunk in response: + chunks.append(chunk) + + # The prompt_filter chunk should be forwarded with choices=[] + assert len(chunks[0].choices) == 0, ( + f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices" + ) + + # At least one chunk must have role='assistant' in its delta + has_role = any( + len(c.choices) > 0 + and getattr(c.choices[0].delta, "role", None) == "assistant" + for c in chunks + ) + assert has_role, ( + "No chunk contained role='assistant' in delta (issue #24221). " + "Chunk deltas: " + + str([ + c.choices[0].delta if c.choices else "no choices" + for c in chunks + ]) + ) + + def test_gemini_legacy_vertex_stop_finish_reason_normalised(): """ The legacy vertex_ai SDK streaming path sets finish_reason from a proto enum diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 2df17e24fd1..42efde90926 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -28,6 +28,7 @@ from litellm.types.utils import ( Function, Message, ModelResponse, + ModelResponseStream, StreamingChoices, Usage, ) @@ -2209,9 +2210,170 @@ class TestTranslateAnthropicOutputFormatToOpenAI: assert sorted(schema["required"]) == ["age", "email", "name"] def test_invalid_output_format_returns_none(self): - assert self.adapter.translate_anthropic_output_format_to_openai("invalid") is None - assert self.adapter.translate_anthropic_output_format_to_openai({"type": "text"}) is None - assert self.adapter.translate_anthropic_output_format_to_openai({"type": "json_schema"}) is None + assert ( + self.adapter.translate_anthropic_output_format_to_openai("invalid") is None + ) + assert ( + self.adapter.translate_anthropic_output_format_to_openai({"type": "text"}) + is None + ) + assert ( + self.adapter.translate_anthropic_output_format_to_openai( + {"type": "json_schema"} + ) + is None + ) + + +class TestAnthropicStreamWrapperToolArgs: + """ + Regression test for https://github.com/BerriAI/litellm/issues/24134 + + When Gemini sends tool call args in the same streaming chunk as a content + block transition, the Anthropic adapter was discarding the processed_chunk + containing input_json_delta. This verifies the args are preserved. + """ + + def _build_chunks(self): + """Build mock OpenAI-format chunks simulating Gemini tool call response.""" + # Chunk 1: text content + text_chunk = ModelResponseStream( + id="chatcmpl-123", + created=1700000000, + model="gemini-2.0-flash", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content="Let me check", role="assistant"), + finish_reason=None, + ) + ], + ) + + # Chunk 2: tool call (triggers new content block + carries args) + tool_chunk = ModelResponseStream( + id="chatcmpl-123", + created=1700000000, + model="gemini-2.0-flash", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + tool_calls=[ + ChatCompletionDeltaToolCall( + id="call_123", + type="function", + function=Function( + name="get_weather", + arguments='{"city": "Tokyo"}', + ), + index=0, + ) + ] + ), + finish_reason=None, + ) + ], + ) + + # Chunk 3: finish + finish_chunk = ModelResponseStream( + id="chatcmpl-123", + created=1700000000, + model="gemini-2.0-flash", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta(), + finish_reason="stop", + ) + ], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + + return [text_chunk, tool_chunk, finish_chunk] + + def _make_stream_wrapper(self, chunks): + from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( + AnthropicStreamWrapper, + ) + + class SimpleIterator: + def __init__(self, items): + self._items = iter(items) + + def __iter__(self): + return self + + def __next__(self): + return next(self._items) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._items) + except StopIteration: + raise StopAsyncIteration + + return AnthropicStreamWrapper( + completion_stream=SimpleIterator(chunks), + model="gemini/gemini-2.0-flash", + ) + + def _find_tool_deltas(self, events): + return [ + e for e in events + if isinstance(e, dict) + and e.get("type") == "content_block_delta" + and isinstance(e.get("delta"), dict) + and e["delta"].get("type") == "input_json_delta" + ] + + def test_sync_tool_args_not_dropped(self): + import json + + chunks = self._build_chunks() + wrapper = self._make_stream_wrapper(chunks) + + events = list(wrapper) + tool_deltas = self._find_tool_deltas(events) + + assert len(tool_deltas) > 0, ( + f"No input_json_delta events found (issue #24134). " + f"Event types: {[e.get('type') for e in events if isinstance(e, dict)]}" + ) + + combined = "".join(d["delta"]["partial_json"] for d in tool_deltas) + parsed = json.loads(combined) + assert parsed == {"city": "Tokyo"} + + @pytest.mark.asyncio + async def test_async_tool_args_not_dropped(self): + import json + + chunks = self._build_chunks() + wrapper = self._make_stream_wrapper(chunks) + + events = [] + async for event in wrapper: + events.append(event) + + tool_deltas = self._find_tool_deltas(events) + + assert len(tool_deltas) > 0, ( + f"No input_json_delta events found (issue #24134). " + f"Event types: {[e.get('type') for e in events if isinstance(e, dict)]}" + ) + + combined = "".join(d["delta"]["partial_json"] for d in tool_deltas) + parsed = json.loads(combined) + assert parsed == {"city": "Tokyo"} + def test_translate_anthropic_tool_choice_none(): diff --git a/tests/test_litellm/llms/gemini/test_cost_calculator.py b/tests/test_litellm/llms/gemini/test_cost_calculator.py new file mode 100644 index 00000000000..9bb83aa7cff --- /dev/null +++ b/tests/test_litellm/llms/gemini/test_cost_calculator.py @@ -0,0 +1,65 @@ +import pytest + +from litellm.llms.gemini.cost_calculator import cost_per_web_search_request +from litellm.types.utils import PromptTokensDetailsWrapper, Usage + + +def _make_usage(web_search_requests: int) -> Usage: + return Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + prompt_tokens_details=PromptTokensDetailsWrapper( + web_search_requests=web_search_requests, + ), + ) + + +def test_per_query_billing(): + """web_search_billing_unit=per_query charges per search query.""" + model_info = { + "key": "gemini/gemini-3-flash-preview", + "web_search_billing_unit": "per_query", + "search_context_cost_per_query": { + "search_context_size_medium": 0.014, + }, + } + cost = cost_per_web_search_request(usage=_make_usage(3), model_info=model_info) + assert cost == pytest.approx(0.014 * 3) + + +def test_per_prompt_billing(): + """web_search_billing_unit=per_prompt (default) clamps to 1.""" + model_info = { + "key": "gemini/gemini-2.5-flash", + "search_context_cost_per_query": { + "search_context_size_medium": 0.035, + }, + } + cost = cost_per_web_search_request(usage=_make_usage(3), model_info=model_info) + assert cost == pytest.approx(0.035 * 1) + + +def test_default_billing_unit_is_per_prompt(): + """Without web_search_billing_unit, defaults to per_prompt (clamp to 1).""" + model_info = {"key": "gemini/gemini-2.0-flash"} + cost = cost_per_web_search_request(usage=_make_usage(2), model_info=model_info) + assert cost == pytest.approx(0.035 * 1) + + +def test_zero_requests(): + """Zero web search requests should return zero cost.""" + model_info = { + "key": "gemini/gemini-3-flash-preview", + "web_search_billing_unit": "per_query", + } + cost = cost_per_web_search_request(usage=_make_usage(0), model_info=model_info) + assert cost == 0.0 + + +def test_no_usage_details(): + """Missing prompt_tokens_details should return zero cost.""" + model_info = {"key": "gemini/gemini-3-flash-preview"} + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + cost = cost_per_web_search_request(usage=usage, model_info=model_info) + assert cost == 0.0 diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index aebab33e808..6383bfc9e18 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -1031,3 +1031,131 @@ def test_gpt5_1_logprobs_dropped_with_reasoning_effort(config: OpenAIConfig): assert "logprobs" not in params assert "top_p" not in params assert params["reasoning_effort"] == "high" + + +# --------------------------------------------------------------------------- +# Responses API: GPT-5 temperature validation (#16090) +# --------------------------------------------------------------------------- + +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams + + +@pytest.fixture() +def responses_config() -> OpenAIResponsesAPIConfig: + return OpenAIResponsesAPIConfig() + + +def test_responses_gpt5_drop_temperature( + responses_config: OpenAIResponsesAPIConfig, +): + """drop_params=True should silently drop temperature!=1 for gpt-5.""" + params = responses_config.map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + temperature=0.5, + ), + model="gpt-5", + drop_params=True, + ) + assert "temperature" not in params + + +def test_responses_gpt5_reject_temperature( + responses_config: OpenAIResponsesAPIConfig, +): + """Without drop_params, temperature!=1 should raise UnsupportedParamsError.""" + with pytest.raises(litellm.UnsupportedParamsError): + responses_config.map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + temperature=0.5, + ), + model="gpt-5", + drop_params=False, + ) + + +def test_responses_gpt5_allow_temperature_1( + responses_config: OpenAIResponsesAPIConfig, +): + """temperature=1 should always be allowed for gpt-5.""" + params = responses_config.map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + temperature=1, + ), + model="gpt-5", + drop_params=False, + ) + assert params["temperature"] == 1 + + +def test_responses_gpt5_mini_drop_temperature( + responses_config: OpenAIResponsesAPIConfig, +): + """gpt-5-mini should also drop temperature!=1.""" + params = responses_config.map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + temperature=0.7, + ), + model="gpt-5-mini", + drop_params=True, + ) + assert "temperature" not in params + + +def test_responses_gpt5_chat_allow_temperature( + responses_config: OpenAIResponsesAPIConfig, +): + """gpt-5-chat models should allow any temperature (not GPT-5 restricted).""" + params = responses_config.map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + temperature=0.3, + ), + model="gpt-5-chat-latest", + drop_params=False, + ) + assert params["temperature"] == 0.3 + + +def test_responses_gpt51_allow_temperature_no_reasoning( + responses_config: OpenAIResponsesAPIConfig, +): + """gpt-5.1 supports reasoning_effort='none'; no reasoning defaults to 'none', + so temperature should be allowed.""" + params = responses_config.map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + temperature=0.5, + ), + model="gpt-5.1", + drop_params=False, + ) + assert params["temperature"] == 0.5 + + +def test_responses_gpt51_drop_temperature_with_high_effort( + responses_config: OpenAIResponsesAPIConfig, +): + """gpt-5.1 with reasoning.effort='high' should drop temperature!=1.""" + params = responses_config.map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + temperature=0.5, + reasoning={"effort": "high"}, + ), + model="gpt-5.1", + drop_params=True, + ) + assert "temperature" not in params + + +def test_responses_gpt54_allow_temperature_effort_none( + responses_config: OpenAIResponsesAPIConfig, +): + """gpt-5.4 with explicit reasoning.effort='none' should allow temperature.""" + params = responses_config.map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + temperature=0.7, + reasoning={"effort": "none"}, + ), + model="gpt-5.4", + drop_params=False, + ) + assert params["temperature"] == 0.7 diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index c475d6461b8..dc344a433bc 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -813,6 +813,10 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): }, "additionalProperties": False, }, + "web_search_billing_unit": { + "type": "string", + "enum": ["per_prompt", "per_query"], + }, "citation_cost_per_token": {"type": "number"}, "supported_modalities": { "type": "array",