From 9cf09c18fefd5ed25c6d25d0f0e3b4acdadf71f6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 16 Oct 2026 15:23:46 -0700 Subject: [PATCH] fix(providers): route Azure Responses past dated api versions, count Grok reasoning tokens on azure_ai, refresh provider sets after register_model, fix gpt-6.1-sol Azure rows (#46377) * fix(azure): add responses_api_version so the Responses route stops using the deployment chat api_version * fix(azure_ai): fold Grok reasoning tokens into completion_tokens like the xai provider * fix(utils): refresh provider model sets after register_model * fix(cost-map): align azure gpt-6.1-sol reasoning effort flags with the openai row * test(e2e): cover Azure Responses on a deployment pinned to a dated chat api_version * fix(utils): keep router deployment ids out of provider model sets register_model now refreshes the provider sets, so the per-request and router registrations keyed by a deployment id put those opaque ids into wildcard model listings. Those callers now pass add_to_provider_model_sets=False. * fix(azure): carry responses_api_version through the chat to Responses bridge completion() dropped responses_api_version from the litellm_params it hands the Responses bridge, so azure/responses/ on a dated chat api_version still built the 404ing /openai/responses URL. Also cover the auto-router marker never forwarding the new param. * fix(azure): stop listing a wrong set of reasoning efforts when none is rejected The message said only low, medium and high are supported, which is wrong for models that also take xhigh such as gpt-6.1-sol. * test(azure): use the latest grok and gpt-5.4 rows in the new azure tests * fix(proxy): list each wildcard-expanded model once A provider set can hold a model both bare and provider-prefixed, which now happens after any priced request registers its openai/ key, so the openai/* listing showed that model twice. * fix(azure): route Responses past dated GA api_versions by default A deployment pinned to a dated GA api_version such as 2024-10-21 for chat now sends Responses calls (create, get, delete, cancel, input_items and the azure/responses/ bridge) to the default Responses version, because those dated versions have no Responses route on Azure and 404. responses_api_version still overrides it, -preview, v1, latest and preview versions are kept, an api-version in the api_base query still wins, and chat keeps api_version as before * chore(azure_ai): mark the grok chunk_parser dict as an override contract * fix(azure): rewrite only the route's /openai segment when moving a call to v1 The v1 rewrite in the Azure URL builder replaced every "/openai" substring in the path, so a gateway api_base like https://apim.example.com/openai-gw/team turned into /openai/v1-gw/team/openai/v1/responses. Dated GA Responses calls now land on the v1 route by default, which made this reachable for them too, so the rewrite now only touches the last /openai path segment. The Responses version resolver no longer duplicates the builder's default for an unset api_version. Tests cover the gateway api_base case, the env AZURE_API_BASE query-string precedence, and the e2e suite now has one test for the dated GA default and one where only responses_api_version can make the call work * test(integration): cover Azure Responses api_version, Grok reasoning spend, wildcard listing and reasoning_effort none Wire tests pin the Azure route and query for chat, Responses and the azure/responses bridge on a dated GA api_version, and check responses_api_version never reaches the request body. Cost cases check azure_ai Grok folds reasoning tokens into completion_tokens and spend, streaming and not. A config-driven proxy checks /v1/models next to an openai/* wildcard lists each id once and never a deployment id. Azure gpt-6.1-sol rejects reasoning_effort none before reaching upstream and drops it with drop_params * fix(utils): list ai21 chat and vertex code models under their provider wildcard register_model now files ai21 chat models into ai21_chat_models, which models_by_provider never read, so a registered ai21 chat model dropped out of ai21/* listings. models_by_provider now reads ai21_chat_models and the vertex code chat and code text sets too. * fix(main): keep caller-chosen model names out of provider wildcard listings --------- Co-authored-by: yuneng-jiang --- litellm/__init__.py | 4 +- .../litellm_core_utils/get_litellm_params.py | 1 + .../llms/azure/chat/gpt_5_transformation.py | 1 - litellm/llms/azure/common_utils.py | 10 +- .../llms/azure/responses/transformation.py | 39 ++- litellm/llms/azure_ai/chat/transformation.py | 36 ++- litellm/main.py | 14 +- ...odel_prices_and_context_window_backup.json | 12 +- litellm/proxy/auth/model_checks.py | 2 +- litellm/router.py | 5 +- litellm/types/litellm_params.py | 1 + litellm/types/router.py | 2 + litellm/utils.py | 59 +--- model_prices_and_context_window.json | 12 +- .../e2e/llm_translation/responses_helpers.py | 5 +- .../e2e/llm_translation/test_responses_e2e.py | 66 +++- tests/e2e/models.py | 1 + .../cost_calculation/cases/misc.py | 57 ++++ .../integration/cost_calculation/cost_map.py | 6 + .../test_azure_reasoning_effort_none_wire.py | 58 ++++ .../test_azure_responses_api_version_wire.py | 108 +++++++ .../routing/test_wildcard_model_listing.py | 71 +++++ .../test_azure_chat_gpt_transformation.py | 30 ++ .../response/test_azure_transformation.py | 282 +++++++++++++++++- .../chat/test_azure_ai_transformation.py | 74 +++++ .../test_openai_responses_transformation.py | 4 +- tests/unit/proxy/auth/test_model_checks.py | 20 ++ .../router_strategy/test_complexity_router.py | 2 + tests/unit/test_model_prices_schema.py | 18 ++ tests/unit/test_utils.py | 81 +++++ tests/unit/types/test_litellm_params.py | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 32 files changed, 997 insertions(+), 89 deletions(-) create mode 100644 tests/integration/providers/test_azure_reasoning_effort_none_wire.py create mode 100644 tests/integration/providers/test_azure_responses_api_version_wire.py create mode 100644 tests/integration/routing/test_wildcard_model_listing.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 83e7642c39f..df52808643d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1177,7 +1177,9 @@ def _build_models_by_provider() -> dict: "edenai": edenai_models, "datarobot": datarobot_models, "vertex_ai": vertex_chat_models + | vertex_code_chat_models | vertex_text_models + | vertex_code_text_models | vertex_anthropic_models | vertex_vision_models | vertex_language_models @@ -1185,7 +1187,7 @@ def _build_models_by_provider() -> dict: | vertex_minimax_models | vertex_moonshot_models | vertex_zai_models, - "ai21": ai21_models, + "ai21": ai21_models | ai21_chat_models, "bedrock": bedrock_models | bedrock_converse_models, "petals": petals_models, "ollama": ollama_models, diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 04755f9e1e9..9dee3714c60 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -78,6 +78,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "github_copilot_auth_type", "github_copilot_user_session", "fireworks_forward_user_id", + "responses_api_version", PROVIDER_AFFINITY_HEADER_KWARG_KEY, } ) diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index 82150e415aa..a23fffb5744 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -95,7 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): status_code=400, message=( "Azure OpenAI does not support reasoning_effort='none' for this model. " - "Supported values are: 'low', 'medium', and 'high'. " "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n" "`litellm_settings:\n drop_params: true`\n" "Issue: https://github.com/BerriAI/litellm/issues/16704" diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 3564d7a1364..46106026920 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -2,6 +2,7 @@ import asyncio import hashlib import json import os +import re from collections.abc import Callable, Mapping from functools import lru_cache from types import MappingProxyType @@ -25,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.utils import add_path_to_api_base azure_ad_cache: Final = DualCache() +_LAST_OPENAI_PATH_SEGMENT: Final = re.compile(r"^(.*/openai)(?=/|$)") class _AzureAdTokenJson(TypedDict, total=False): @@ -798,6 +800,10 @@ class BaseAzureLLM(BaseOpenAILLM): ) -> dict[str, str]: # mutable-ok: mirrors override contract return cls._base_validate_azure_environment(headers, litellm_params) + @staticmethod + def resolve_azure_api_base(api_base: str | None) -> str | None: + return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") + @staticmethod def _get_base_azure_url( api_base: str | None, @@ -815,7 +821,7 @@ class BaseAzureLLM(BaseOpenAILLM): default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route. """ - api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") + api_base = BaseAzureLLM.resolve_azure_api_base(api_base) if api_base is None: raise ValueError( f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`" @@ -843,7 +849,7 @@ class BaseAzureLLM(BaseOpenAILLM): # ensure the request go to /openai/v1 and not just /openai if "/openai/v1" not in new_url: parsed_url: Final = httpx.URL(new_url) - new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1"))) + new_url = str(parsed_url.copy_with(path=_LAST_OPENAI_PATH_SEGMENT.sub(r"\1/v1", parsed_url.path, 1))) # Use the new query_params dictionary final_url: Final = httpx.URL(new_url).copy_with(params=query_params) diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 98b4241d7ab..75714d5253b 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -1,4 +1,7 @@ +import re +from collections.abc import Mapping from copy import deepcopy +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -6,6 +9,7 @@ from openai.types.responses import ResponseReasoningItem from pydantic import TypeAdapter from litellm._logging import verbose_logger +from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import BaseAzureLLM @@ -25,6 +29,34 @@ else: _DECODED_JSON: Final = TypeAdapter(object) +_DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}") + + +def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None: + value: Final = litellm_params.get(key) + return value if isinstance(value, str) and value else None + + +def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> str | None: + """ + Dated GA api_versions (e.g. 2024-10-21) predate the Responses API on Azure and 404 there, so a deployment + pinned to one for chat gets the default Responses version unless it sets responses_api_version. + """ + api_version: Final = _str_param(litellm_params, "api_version") + if api_base is not None and "api-version" in httpx.URL(api_base).params: + return api_version + responses_api_version: Final = _str_param(litellm_params, "responses_api_version") + if responses_api_version is not None: + return responses_api_version + if api_version is not None and _DATED_GA_API_VERSION.fullmatch(api_version): + return AZURE_DEFAULT_RESPONSES_API_VERSION + return api_version + + +def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)}) + + class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): # Parameters not supported by Azure Responses API AZURE_UNSUPPORTED_PARAMS = ["context_management"] @@ -170,11 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): - A complete URL string, e.g., "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview" """ - from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION - + resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base) return BaseAzureLLM.get_base_azure_url( - api_base=api_base, - litellm_params=litellm_params, + api_base=resolved_api_base, + litellm_params=_with_responses_api_version(resolved_api_base, litellm_params), route="/openai/responses", default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION, ) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index a8e6320bd1e..fc22e9a8b73 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -1,6 +1,7 @@ import copy import enum import re +from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Final, cast import httpx @@ -21,12 +22,12 @@ from litellm.llms.azure_ai.common_utils import ( from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error -from litellm.llms.openai.openai import OpenAIConfig +from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig from litellm.llms.xai.chat.transformation import XAIChatConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import ModelResponse, ProviderField +from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField from litellm.utils import add_path_to_api_base, supports_tool_choice if TYPE_CHECKING: @@ -45,6 +46,18 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ( ) +def _is_grok_model(model: str) -> bool: + return "grok" in model + + +class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator): + def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract + usage: Final = chunk.get("usage") + if _is_grok_model(str(chunk.get("model") or "")) and isinstance(usage, dict): + XAIChatConfig.fold_reasoning_tokens_into_completion(usage) + return super().chunk_parser(chunk) + + class AzureAIGPT5Config(OpenAIGPT5Config): @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -114,7 +127,7 @@ class AzureAIStudioConfig(OpenAIConfig): """ Check if the model supports stop tokens. """ - if "grok" in model: + if _is_grok_model(model): # Reuse Xai method for Grok model xai_config: Final = XAIChatConfig() return xai_config.supports_stop_reason(model) @@ -319,7 +332,7 @@ class AzureAIStudioConfig(OpenAIConfig): json_mode: bool | None = None, ) -> ModelResponse: model_response.model = f"azure_ai/{model}" - return super().transform_response( + response: Final = super().transform_response( model=model, raw_response=raw_response, model_response=model_response, @@ -332,6 +345,21 @@ class AzureAIStudioConfig(OpenAIConfig): api_key=api_key, json_mode=json_mode, ) + if _is_grok_model(model): + XAIChatConfig.fold_reasoning_tokens_into_completion(response) + return response + + def get_model_response_iterator( + self, + streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, + sync_stream: bool, + json_mode: bool | None = False, + ) -> "AzureAIChatCompletionResponseIterator": + return AzureAIChatCompletionResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) def should_retry_llm_api_inside_llm_translation_on_http_error( self, e: httpx.HTTPStatusError, litellm_params: dict diff --git a/litellm/main.py b/litellm/main.py index e6caa5c25b6..3646b48b8cc 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1212,13 +1212,18 @@ def _register_custom_pricing_for_request( litellm.register_model({shared_key: entry}, persist_across_reloads=False) return litellm.register_model( - { - deployment_id: entry, - shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry), - }, + {deployment_id: entry}, persist_across_reloads=False, warning_display_name=shared_key, custom_llm_provider=custom_llm_provider, + add_to_provider_model_sets=False, + ) + litellm.register_model( + {shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)}, + persist_across_reloads=False, + warning_display_name=shared_key, + custom_llm_provider=custom_llm_provider, + add_to_provider_model_sets=False, ) @@ -5785,6 +5790,7 @@ def completion( "github_copilot_user_session", PROVIDER_AFFINITY_HEADER_KWARG_KEY, "fireworks_forward_user_id", + "responses_api_version", ) if key in kwargs }, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e811045bf1a..47208d26e2a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4029,7 +4029,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_cache_breakpoint": true, @@ -8698,7 +8698,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8747,7 +8747,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8796,7 +8796,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8845,7 +8845,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8894,7 +8894,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index b70cdb08b96..8a7b6dab419 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -332,7 +332,7 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ else: model = f"{wildcard_provider_prefix}/{model}" suffix_appended_wildcard_models.append(model) - return suffix_appended_wildcard_models or [] + return list(dict.fromkeys(suffix_appended_wildcard_models)) def expand_wildcard_deployments_for_model_info( diff --git a/litellm/router.py b/litellm/router.py index d041f958e17..99adc6c5fa6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -501,7 +501,9 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _CallbackT = TypeVar("_CallbackT") -_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) +_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset( + {"model", "api_base", "api_key", "api_version", "responses_api_version"} +) _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" _CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") _CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600 @@ -10995,6 +10997,7 @@ class Router: persist_across_reloads=False, warning_display_name=model, custom_llm_provider=custom_llm_provider, + add_to_provider_model_sets=False, ) ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 8387afa22bb..174d07a2cea 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -67,6 +67,7 @@ class ProviderConnection: api_key: str | None = None api_base: str | None = None api_version: str | None = None + responses_api_version: str | None = None region_name: str | None = None headers: Mapping[str, str] | None = None provider_specific_header: "ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None" = None diff --git a/litellm/types/router.py b/litellm/types/router.py index 0ca39e2400d..8f3ba7c0a8d 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -496,6 +496,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): def __init__(self, /, **data: object) -> None: ... # kwargs-ok: provider parameters vary by deployment custom_llm_provider: str | None = None + responses_api_version: str | None = None tpm: int | None = None rpm: int | None = None itpm: int | None = None @@ -703,6 +704,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): api_key: str | None api_base: str | None api_version: str | None + responses_api_version: ReadOnly[str | None] timeout: float | str | httpx.Timeout | SDKTimeout | None # writable-ok: preserve timeout updates stream_timeout: float | str | None max_retries: int | None diff --git a/litellm/utils.py b/litellm/utils.py index 76ef50c3a2b..3b43c570466 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3445,6 +3445,7 @@ def register_model( persist_across_reloads: bool = True, warning_display_name: str | None = None, custom_llm_provider: str | None = None, + add_to_provider_model_sets: bool = True, ): """ Register new / Override existing models (and their pricing) to specific providers. @@ -3473,6 +3474,10 @@ def register_model( ``custom_llm_provider`` scopes the built-in cost-map match to entries for that provider, so a deployment id that happens to equal another provider's catalog key stays its own provider-less entry instead of merging into it. + + ``add_to_provider_model_sets`` adds the keys to the per-provider model sets + behind provider resolution and wildcard model listing. Pass False when the + keys are opaque ids rather than model names (e.g. router deployment ids). """ loaded_model_cost = {} @@ -3558,56 +3563,10 @@ def register_model( _invalidate_model_cost_lowercase_map() verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", model_cost_key, model_cost_key) - # add new model names to provider lists - if value.get("litellm_provider") == "openai": - if key not in litellm.open_ai_chat_completion_models: - litellm.open_ai_chat_completion_models.add(key) - elif value.get("litellm_provider") == "text-completion-openai": - if key not in litellm.open_ai_text_completion_models: - litellm.open_ai_text_completion_models.add(key) - elif value.get("litellm_provider") == "cohere": - if key not in litellm.cohere_models: - litellm.cohere_models.add(key) - elif value.get("litellm_provider") == "anthropic": - if key not in litellm.anthropic_models: - litellm.anthropic_models.add(key) - elif value.get("litellm_provider") == "openrouter": - split_string = key.split("/", 1) - if split_string[-1] not in litellm.openrouter_models: - litellm.openrouter_models.add(split_string[-1]) - elif value.get("litellm_provider") == "vercel_ai_gateway": - if key not in litellm.vercel_ai_gateway_models: - litellm.vercel_ai_gateway_models.add(key) - elif value.get("litellm_provider") == "edenai": - if key not in litellm.edenai_models: - litellm.edenai_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-text-models": - if key not in litellm.vertex_text_models: - litellm.vertex_text_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-code-text-models": - if key not in litellm.vertex_code_text_models: - litellm.vertex_code_text_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-chat-models": - if key not in litellm.vertex_chat_models: - litellm.vertex_chat_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-code-chat-models": - if key not in litellm.vertex_code_chat_models: - litellm.vertex_code_chat_models.add(key) - elif value.get("litellm_provider") == "ai21": - if key not in litellm.ai21_models: - litellm.ai21_models.add(key) - elif value.get("litellm_provider") == "nlp_cloud": - if key not in litellm.nlp_cloud_models: - litellm.nlp_cloud_models.add(key) - elif value.get("litellm_provider") == "aleph_alpha": - if key not in litellm.aleph_alpha_models: - litellm.aleph_alpha_models.add(key) - elif value.get("litellm_provider") == "bedrock": - if key not in litellm.bedrock_models: - litellm.bedrock_models.add(key) - elif value.get("litellm_provider") == "novita": - if key not in litellm.novita_models: - litellm.novita_models.add(key) + if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter": + litellm.openrouter_models.add(_key_str.split("/", 1)[-1]) + if add_to_provider_model_sets: + litellm.add_known_models(loaded_model_cost) return model_cost diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e811045bf1a..47208d26e2a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4029,7 +4029,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_cache_breakpoint": true, @@ -8698,7 +8698,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8747,7 +8747,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8796,7 +8796,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8845,7 +8845,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8894,7 +8894,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/tests/e2e/llm_translation/responses_helpers.py b/tests/e2e/llm_translation/responses_helpers.py index be0fac74bb6..29b923a7f34 100644 --- a/tests/e2e/llm_translation/responses_helpers.py +++ b/tests/e2e/llm_translation/responses_helpers.py @@ -8,10 +8,13 @@ AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" AZURE_OPENAI_API_VERSION: Final = "v1" -def azure_openai_params(api_version: str = AZURE_OPENAI_API_VERSION) -> LiteLLMParamsBody: +def azure_openai_params( + api_version: str = AZURE_OPENAI_API_VERSION, responses_api_version: str | None = None +) -> LiteLLMParamsBody: return LiteLLMParamsBody( model=AZURE_OPENAI_BACKEND, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY", api_version=api_version, + responses_api_version=responses_api_version, ) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index ee4699703b2..bb75569ad99 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -40,7 +40,7 @@ from provider_edge import LiveEdge, start_provider_edge from provider_edge_bedrock import ConverseBodyCapture, bedrock_signer from proxy_client import ProxyClient from pydantic import BaseModel, TypeAdapter -from responses_helpers import AZURE_OPENAI_BACKEND, azure_openai_params +from responses_helpers import AZURE_OPENAI_API_VERSION, AZURE_OPENAI_BACKEND, azure_openai_params from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -748,6 +748,70 @@ class TestResponses: f"/responses over azure openai preview returned no output text: {response.output!r}" ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=(AZURE_OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_azure_openai_dated_ga_chat_api_version_serves_responses( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register( + proxy, + resources, + azure_openai_params(api_version="2024-10-21"), + prefix="e2e-responses-azure-dated-chat", + ) + client: Final = sdk.openai(resources.key()) + + response: Final = client.responses.create( + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + extra_body=NO_PROXY_CACHE, + ) + assert response.output_text.strip(), ( + f"/responses over an azure deployment pinned to a dated GA chat api_version returned no output text: " + f"{response.output!r}" + ) + + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=(AZURE_OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_azure_openai_responses_api_version_overrides_a_pre_responses_api_version( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + # Azure rejects /openai/responses below api-version 2025-03-01-preview (probed live 2026-10-15), + # so this call only succeeds if the proxy sends responses_api_version instead of api_version. + model: Final = _register( + proxy, + resources, + azure_openai_params(api_version="2024-10-01-preview", responses_api_version=AZURE_OPENAI_API_VERSION), + prefix="e2e-responses-azure-override", + ) + client: Final = sdk.openai(resources.key()) + + response: Final = client.responses.create( + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + extra_body=NO_PROXY_CACHE, + ) + assert response.output_text.strip(), ( + f"/responses over an azure deployment with responses_api_version set returned no output text: " + f"{response.output!r}" + ) + @meta( Subject( domain=Domain.LLM_TRANSLATION, diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 08fc7a79df3..d7a10c6da1f 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1366,6 +1366,7 @@ class LiteLLMParamsBody(BaseModel): litellm_credential_name: str | None = None api_base: str | None = Field(default=None, repr=False) api_version: str | None = None + responses_api_version: str | None = None realtime_protocol: str | None = None allowed_openai_params: list[str] | None = None drop_params: bool | None = None diff --git a/tests/integration/cost_calculation/cases/misc.py b/tests/integration/cost_calculation/cases/misc.py index c9729353c8e..f98dda5fb0c 100644 --- a/tests/integration/cost_calculation/cases/misc.py +++ b/tests/integration/cost_calculation/cases/misc.py @@ -834,6 +834,61 @@ AZURE_AI_GPT_5_4_MINI_LATEST_STREAM: Final = CostTrackingTestCase( ) +# azure_ai/grok-4.6 (Grok on Azure AI Foundry reports reasoning_tokens outside completion_tokens, like xai) +AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION: Final = CostTrackingTestCase( + name="azure-ai-grok-4.6-reasoning_folded_into_completion", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="azure_ai/grok-4.6", + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "azure-ai-grok-4.6-reasoning"}], + "stream": False, + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "model": "grok-4.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2552, + "completion_tokens_details": {"reasoning_tokens": 300}, + }, + }, + ), + expected=ExactExpected( + spend=0.0108, input_cost=0.00368, output_cost=0.00712, prompt_tokens=1840, completion_tokens=712 + ), +) + +AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION_STREAM: Final = CostTrackingTestCase( + name="azure-ai-grok-4.6-reasoning_folded_into_completion-stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="azure_ai/grok-4.6", + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "azure-ai-grok-4.6-reasoning-stream"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + response=SseResponse( + content_type="text/event-stream", + frames=( + 'data: {"id":"chatcmpl-$REQUEST_ID","object":"chat.completion.chunk","created":1,"model":"grok-4.6","choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":null}]}\n\n', + 'data: {"id":"chatcmpl-$REQUEST_ID","object":"chat.completion.chunk","created":1,"model":"grok-4.6","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n', + 'data: {"id":"chatcmpl-$REQUEST_ID","object":"chat.completion.chunk","created":1,"model":"grok-4.6","choices":[],"usage":{"prompt_tokens":1840,"completion_tokens":412,"total_tokens":2552,"completion_tokens_details":{"reasoning_tokens":300}}}\n\n', + "data: [DONE]\n\n", + ), + ), + expected=ExactExpected( + spend=0.0108, input_cost=0.00368, output_cost=0.00712, prompt_tokens=1840, completion_tokens=712 + ), +) + + # azure_ai/claude-haiku-4-5 (Azure AI Foundry Anthropic, web search billed per server_tool_use.web_search_requests) AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH: Final = CostTrackingTestCase( name="azure-ai-claude-haiku-4-5-messages_web_search_two_requests", @@ -1978,6 +2033,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( BEDROCK_INVOKE_HAIKU_STREAM, AZURE_AI_GPT_5_4_MINI_LATEST, AZURE_AI_GPT_5_4_MINI_LATEST_STREAM, + AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION, + AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION_STREAM, AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH, AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH_STREAM, AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH_ONE_REQUEST, diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 5e2b99a113f..2ef42249a7e 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -764,6 +764,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token": 7.5e-07, "output_cost_per_token": 4.5e-06, }, + "azure_ai/grok-4.6": { + "litellm_provider": "azure_ai", + "mode": "chat", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + }, "azure_ai/claude-haiku-4-5": { "litellm_provider": "azure_ai", "mode": "chat", diff --git a/tests/integration/providers/test_azure_reasoning_effort_none_wire.py b/tests/integration/providers/test_azure_reasoning_effort_none_wire.py new file mode 100644 index 00000000000..c95e497c672 --- /dev/null +++ b/tests/integration/providers/test_azure_reasoning_effort_none_wire.py @@ -0,0 +1,58 @@ +from typing import Final + +from integration._support.client import Gateway +from integration._support.openai_wire import chat_reply +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_DEPLOYMENT: Final = "gpt-6.1-sol" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "no reasoning please"}] + + +def _reply(_request: Request) -> Reply: + return chat_reply("chatcmpl-effort-none", _DEPLOYMENT, "answered without reasoning", stream=False) + + +def test_reasoning_effort_none_is_rejected_by_litellm_before_reaching_azure(gateway: Gateway) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"azure/{_DEPLOYMENT}", + api_base=wire.url, + api_key="synthetic-azure-key", + api_version="2025-04-01-preview", + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _MESSAGES, "reasoning_effort": "none"}, + key=scenario.key(), + ) + + assert response.status_code == 400, response.text + assert "reasoning_effort='none'" in response.text, response.text + assert wire.drain() == () + + +def test_reasoning_effort_none_is_dropped_when_the_deployment_drops_params(gateway: Gateway) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"azure/{_DEPLOYMENT}", + api_base=wire.url, + api_key="synthetic-azure-key", + api_version="2025-04-01-preview", + drop_params=True, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _MESSAGES, "reasoning_effort": "none"}, + key=scenario.key(), + ) + + assert response.status_code == 200, response.text + assert "answered without reasoning" in response.text, response.text + received: Final = wire.drain() + assert [request.method for request in received] == ["POST"], [request.target for request in received] + body: Final = _JSON_OBJECT.validate_json(received[0].body) + assert "reasoning_effort" not in body and "reasoning" not in body, body diff --git a/tests/integration/providers/test_azure_responses_api_version_wire.py b/tests/integration/providers/test_azure_responses_api_version_wire.py new file mode 100644 index 00000000000..13c2f48dee2 --- /dev/null +++ b/tests/integration/providers/test_azure_responses_api_version_wire.py @@ -0,0 +1,108 @@ +from typing import Final +from urllib.parse import parse_qs, urlsplit + +from integration._support.client import Gateway, Scenario +from integration._support.openai_wire import chat_reply, responses_reply +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_DEPLOYMENT: Final = "gpt-5.4-nano" +_DATED_GA_API_VERSION: Final = "2024-10-21" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _reply(request: Request) -> Reply: + if urlsplit(request.target).path.endswith("/responses"): + return responses_reply("resp_azure_version", _DEPLOYMENT, "azure answered", stream=False) + return chat_reply("chatcmpl-azure-version", _DEPLOYMENT, "azure answered", stream=False) + + +def _dated_azure_model(scenario: Scenario, wire: Wire, model: str, responses_api_version: str | None = None) -> str: + if responses_api_version is None: + return scenario.model( + model=model, api_base=wire.url, api_key="synthetic-azure-key", api_version=_DATED_GA_API_VERSION + ) + return scenario.model( + model=model, + api_base=wire.url, + api_key="synthetic-azure-key", + api_version=_DATED_GA_API_VERSION, + responses_api_version=responses_api_version, + ) + + +def _only_post(wire: Wire) -> tuple[str, dict[str, str], dict[str, JsonValue]]: + posts: Final = tuple(request for request in wire.drain() if request.method == "POST") + assert len(posts) == 1, [request.target for request in posts] + target: Final = urlsplit(posts[0].target) + query: Final = {name: values[-1] for name, values in parse_qs(target.query).items()} + return target.path, query, _JSON_OBJECT.validate_json(posts[0].body) + + +def test_chat_keeps_the_dated_api_version_and_never_sends_responses_api_version(gateway: Gateway) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = _dated_azure_model(scenario, wire, f"azure/{_DEPLOYMENT}", responses_api_version="preview") + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "dated chat"}]}, + key=scenario.key(), + ) + + assert response.status_code == 200, response.text + path, query, body = _only_post(wire) + assert (path, query) == ( + f"/openai/deployments/{_DEPLOYMENT}/chat/completions", + {"api-version": _DATED_GA_API_VERSION}, + ) + assert body == { + "model": _DEPLOYMENT, + "messages": [{"role": "user", "content": "dated chat"}], + "stream": False, + }, body + + +def test_responses_on_a_dated_ga_deployment_reaches_the_v1_responses_route(gateway: Gateway) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = _dated_azure_model(scenario, wire, f"azure/{_DEPLOYMENT}") + response: Final = gateway.request( + "POST", "/v1/responses", {"model": model, "input": "dated responses"}, key=scenario.key() + ) + + assert response.status_code == 200, response.text + path, query, body = _only_post(wire) + assert (path, query) == ("/openai/v1/responses", {"api-version": "preview"}) + assert body["input"] == "dated responses", body + + +def test_responses_api_version_overrides_the_chat_api_version_and_stays_out_of_the_body(gateway: Gateway) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = _dated_azure_model( + scenario, wire, f"azure/{_DEPLOYMENT}", responses_api_version="2025-04-01-preview" + ) + response: Final = gateway.request( + "POST", "/v1/responses", {"model": model, "input": "override responses"}, key=scenario.key() + ) + + assert response.status_code == 200, response.text + path, query, body = _only_post(wire) + assert (path, query) == ("/openai/responses", {"api-version": "2025-04-01-preview"}) + assert "responses_api_version" not in body and "api_version" not in body, body + + +def test_chat_bridged_through_azure_responses_on_a_dated_ga_deployment_reaches_v1_responses( + gateway: Gateway, +) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = _dated_azure_model(scenario, wire, f"azure/responses/{_DEPLOYMENT}") + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "bridged chat"}]}, + key=scenario.key(), + ) + + assert response.status_code == 200, response.text + assert "azure answered" in response.text, response.text + path, query, _ = _only_post(wire) + assert (path, query) == ("/openai/v1/responses", {"api-version": "preview"}) diff --git a/tests/integration/routing/test_wildcard_model_listing.py b/tests/integration/routing/test_wildcard_model_listing.py new file mode 100644 index 00000000000..967a515ec08 --- /dev/null +++ b/tests/integration/routing/test_wildcard_model_listing.py @@ -0,0 +1,71 @@ +import uuid +from collections import Counter +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(180) + +_PRICED_DEPLOYMENT_ID: Final = f"priced-deployment-{uuid.uuid4().hex}" +_REGISTERED_AI21_CHAT_MODEL: Final = f"jamba-house-{uuid.uuid4().hex}" + + +@pytest.fixture(scope="module") +def candidate(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("wildcard-listing") + with gateway_from_environment() as base: + credentials: Final = {"api_base": f"{base.upstream_url}/v1", "api_key": "synthetic-listing-key"} + config: Final = { + **yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()), + "model_list": [ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", **credentials}}, + { + "model_name": "priced-chat", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + **credentials, + }, + "model_info": {"id": _PRICED_DEPLOYMENT_ID}, + }, + {"model_name": "ai21/*", "litellm_params": {"model": "ai21/*", **credentials}}, + { + "model_name": "house-jamba", + "litellm_params": {"model": f"ai21/{_REGISTERED_AI21_CHAT_MODEL}", **credentials}, + "model_info": {"litellm_provider": "ai21", "mode": "chat"}, + }, + ], + } + path: Final = directory / "wildcard-listing.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +def _listed_ids(gateway: Gateway) -> list[JsonValue]: + data: Final = gateway.get("/v1/models")["data"] + assert isinstance(data, list), data + return [entry["id"] for entry in data if isinstance(entry, dict)] + + +def test_wildcard_listing_after_a_priced_call_names_each_model_once_and_no_deployment_id(candidate: Gateway) -> None: + candidate.chat("priced-chat") + + listed: Final = _listed_ids(candidate) + + assert "openai/gpt-4o-mini" in listed and "priced-chat" in listed, listed + assert {model: count for model, count in Counter(listed).items() if count > 1} == {}, listed + assert [model for model in listed if _PRICED_DEPLOYMENT_ID in str(model)] == [], listed + + +def test_ai21_wildcard_listing_names_a_chat_model_registered_from_the_config(candidate: Gateway) -> None: + listed: Final = _listed_ids(candidate) + + assert f"ai21/{_REGISTERED_AI21_CHAT_MODEL}" in listed, listed diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index 9875cf439cf..f753fb6b8f9 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -753,3 +753,33 @@ def test_azure_deployment_id_selects_chat_completions_endpoint(respx_mock: MockR assert route.call_count == 1 assert route.calls[0].request.url.path == "/openai/deployments/deployment-gpt-4o/chat/completions" assert response.choices[0].message.content == "Hello" + + +@pytest.mark.respx(assert_all_called=True) +def test_responses_api_version_leaves_the_chat_route_and_body_untouched(respx_mock: MockRouter) -> None: + url: Final = "https://example-resource.openai.azure.com/openai/deployments/gpt-5.4-nano/chat/completions" + payload: Final = { + "id": "chatcmpl-dated", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-5.4-nano", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + } + route: Final = respx_mock.post(url__startswith=url).mock(return_value=httpx.Response(200, json=payload)) + messages: Final = [{"role": "user", "content": "Hello"}] + + response: Final = litellm.completion( + model="azure/gpt-5.4-nano", + messages=messages, + api_base="https://example-resource.openai.azure.com", + api_key="azure-test-key", + api_version="2024-12-01-preview", + responses_api_version="preview", + ) + request: Final = route.calls[0].request + + assert route.call_count == 1 + assert request.url.params["api-version"] == "2024-12-01-preview" + assert json.loads(request.content) == {"model": "gpt-5.4-nano", "messages": messages} + assert response.choices[0].message.content == "Hello" diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index fea5304a689..9b4c618c78b 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -1,11 +1,15 @@ import json +from collections.abc import Callable from copy import deepcopy from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +import respx import litellm +from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.azure.responses.o_series_transformation import ( @@ -282,16 +286,25 @@ class TestAzureResponsesAPIConfig: == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest" ) - # Test with date-based version - should use openai/responses - result_date = self.config.get_complete_url( + result_dated_preview = self.config.get_complete_url( api_base=base_url, - litellm_params={"api_version": "2025-01-01"}, + litellm_params={"api_version": "2025-04-01-preview"}, ) assert ( - result_date - == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" + result_dated_preview + == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview" ) + @pytest.mark.parametrize("api_version", ["2024-10-21", "2024-06-01", "2025-01-01"]) + def test_dated_ga_api_version_routes_responses_like_an_unset_one(self, api_version: str): + base_url: Final = "https://litellm8397336933.openai.azure.com" + + dated: Final = self.config.get_complete_url(api_base=base_url, litellm_params={"api_version": api_version}) + unset: Final = self.config.get_complete_url(api_base=base_url, litellm_params={}) + + assert dated == unset + assert httpx.URL(dated).params["api-version"] == AZURE_DEFAULT_RESPONSES_API_VERSION + def test_azure_get_complete_url_with_default_api_version(self): """Test Azure get_complete_url uses default API version when none is provided""" from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION @@ -320,6 +333,64 @@ class TestAzureResponsesAPIConfig: ) assert result_none_version == expected_url + @pytest.mark.parametrize( + ("api_version", "responses_api_version", "expected_path"), + [ + ("2024-10-21", "2025-04-01-preview", "/openai/responses?api-version=2025-04-01-preview"), + ("2025-04-01-preview", "v1", "/openai/v1/responses?api-version=v1"), + (None, "2025-04-01-preview", "/openai/responses?api-version=2025-04-01-preview"), + ], + ) + def test_responses_api_version_wins_over_api_version( + self, api_version: str | None, responses_api_version: str, expected_path: str + ): + base_url: Final = "https://litellm8397336933.openai.azure.com" + + url: Final = self.config.get_complete_url( + api_base=base_url, + litellm_params={"api_version": api_version, "responses_api_version": responses_api_version}, + ) + + assert url == f"{base_url}{expected_path}" + + @pytest.mark.parametrize("responses_api_version", [None, "v1"]) + def test_api_version_in_api_base_query_wins_over_a_dated_ga_api_version(self, responses_api_version: str | None): + api_base: Final = "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview" + + url: Final = self.config.get_complete_url( + api_base=api_base, + litellm_params={"api_version": "2024-10-21", "responses_api_version": responses_api_version}, + ) + + assert url == api_base + + def test_api_version_in_env_api_base_query_wins_over_a_dated_ga_api_version(self, monkeypatch: pytest.MonkeyPatch): + env_api_base: Final = ( + "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview" + ) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setenv("AZURE_API_BASE", env_api_base) + + url: Final = self.config.get_complete_url(api_base=None, litellm_params={"api_version": "2024-10-21"}) + + assert url == env_api_base + + @pytest.mark.parametrize( + ("api_base", "expected_path"), + [ + ("https://apim.example.com/openai-gw/team", "/openai-gw/team/openai/v1/responses"), + ("https://apim.example.com/openai/team-a", "/openai/team-a/openai/v1/responses"), + ], + ) + @pytest.mark.parametrize("litellm_params", [{"api_version": "2024-10-21"}, {"responses_api_version": "v1"}]) + def test_v1_route_only_rewrites_the_responses_route_segment_of_a_gateway_api_base( + self, api_base: str, expected_path: str, litellm_params: dict[str, str] + ): + url: Final = httpx.URL(self.config.get_complete_url(api_base=api_base, litellm_params=litellm_params)) + + assert url.host == "apim.example.com" + assert url.path == expected_path + def test_azure_cancel_response_api_request(self): """Test Azure cancel response API request transformation""" from litellm.types.router import GenericLiteLLMParams @@ -684,6 +755,207 @@ def test_azure_responses_gpt6_astra_rejects_temperature_while_reasoning(local_mo ) +_COMPLETED_RESPONSE: Final = { + "id": "resp_dated_chat_deployment", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "gpt-5.4-nano", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "truncation": None, + "user": None, +} + + +_AZURE_RESOURCE: Final = "https://example-resource.openai.azure.com" + + +def _unset_api_version_responses_url() -> httpx.URL: + return httpx.URL(AzureOpenAIResponsesAPIConfig().get_complete_url(api_base=_AZURE_RESOURCE, litellm_params={})) + + +def _azure_responses_create(api_version: str) -> str | None: + response: Final = litellm.responses( + model="azure/gpt-5.4-nano", + input="hi", + api_base=_AZURE_RESOURCE, + api_key="azure-test-key", + api_version=api_version, + ) + return response.output[0].content[0].text + + +def _azure_chat_bridged_to_responses(api_version: str) -> str | None: + response: Final = litellm.completion( + model="azure/responses/gpt-5.4-nano", + messages=[{"role": "user", "content": "hi"}], + api_base=_AZURE_RESOURCE, + api_key="azure-test-key", + api_version=api_version, + ) + return response.choices[0].message.content + + +@pytest.mark.parametrize("call_responses", [_azure_responses_create, _azure_chat_bridged_to_responses]) +def test_dated_ga_api_version_sends_responses_to_the_unset_api_version_url( + respx_mock: respx.MockRouter, call_responses: Callable[[str], str | None] +) -> None: + route: Final = respx_mock.post(host="example-resource.openai.azure.com").mock( + return_value=httpx.Response(200, json=_COMPLETED_RESPONSE) + ) + + text: Final = call_responses("2024-10-21") + + assert text == "Done." + assert route.call_count == 1 + assert route.calls[0].request.url == _unset_api_version_responses_url() + + +_LIFECYCLE_BODIES: Final = { + "get": _COMPLETED_RESPONSE, + "delete": {"id": "resp_dated", "object": "response", "deleted": True}, + "cancel": {**_COMPLETED_RESPONSE, "status": "cancelled"}, + "input_items": {"object": "list", "data": [], "first_id": None, "last_id": None, "has_more": False}, +} + + +def _call_lifecycle(operation: str) -> object: + kwargs: Final = { + "response_id": "resp_dated", + "custom_llm_provider": "azure", + "api_base": _AZURE_RESOURCE, + "api_key": "azure-test-key", + "api_version": "2024-10-21", + } + match operation: + case "get": + return litellm.get_responses(**kwargs) + case "delete": + return litellm.delete_responses(**kwargs) + case "cancel": + return litellm.cancel_responses(**kwargs) + case _: + return litellm.list_input_items(**kwargs) + + +@pytest.mark.parametrize( + ("operation", "path_suffix"), + [("get", ""), ("delete", ""), ("cancel", "/cancel"), ("input_items", "/input_items")], +) +def test_dated_ga_api_version_sends_response_lifecycle_calls_to_the_unset_api_version_route( + respx_mock: respx.MockRouter, operation: str, path_suffix: str +) -> None: + route: Final = respx_mock.route(host="example-resource.openai.azure.com").mock( + return_value=httpx.Response(200, json=_LIFECYCLE_BODIES[operation]) + ) + + _call_lifecycle(operation) + + unset_url: Final = _unset_api_version_responses_url() + sent_url: Final = route.calls[0].request.url + assert route.call_count == 1 + assert sent_url.path == f"{unset_url.path}/resp_dated{path_suffix}" + assert sent_url.params["api-version"] == unset_url.params["api-version"] + + +def test_chat_completion_keeps_the_dated_ga_api_version(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post( + f"{_AZURE_RESOURCE}/openai/deployments/gpt-5.4-nano/chat/completions", params={"api-version": "2024-10-21"} + ).mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-dated", + "object": "chat.completion", + "created": 1734366691, + "model": "gpt-5.4-nano", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Done."}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.completion( + model="azure/gpt-5.4-nano", + messages=[{"role": "user", "content": "hi"}], + api_base=_AZURE_RESOURCE, + api_key="azure-test-key", + api_version="2024-10-21", + ) + + assert response.choices[0].message.content == "Done." + assert route.call_count == 1 + + +@pytest.mark.respx(assert_all_called=True) +def test_responses_api_version_routes_a_dated_chat_deployment_to_its_own_responses_version( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post( + "https://example-resource.openai.azure.com/openai/responses", params={"api-version": "2025-04-01-preview"} + ).mock(return_value=httpx.Response(200, json=_COMPLETED_RESPONSE)) + + response: Final = litellm.responses( + model="azure/gpt-5.4-nano", + input="hi", + api_base="https://example-resource.openai.azure.com", + api_key="azure-test-key", + api_version="2024-10-21", + responses_api_version="2025-04-01-preview", + ) + request_body: Final = json.loads(route.calls[0].request.content) + + assert route.call_count == 1 + assert response.output[0].content[0].text == "Done." + assert request_body["model"] == "gpt-5.4-nano" + assert "responses_api_version" not in request_body + assert "api_version" not in request_body + + +@pytest.mark.respx(assert_all_called=True) +def test_responses_api_version_reaches_the_url_when_chat_completion_bridges_to_responses( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post( + "https://example-resource.openai.azure.com/openai/v1/responses", params={"api-version": "v1"} + ).mock(return_value=httpx.Response(200, json=_COMPLETED_RESPONSE)) + + response: Final = litellm.completion( + model="azure/responses/gpt-5.4-nano", + messages=[{"role": "user", "content": "hi"}], + api_base="https://example-resource.openai.azure.com", + api_key="azure-test-key", + api_version="2024-10-21", + responses_api_version="v1", + ) + request_body: Final = json.loads(route.calls[0].request.content) + + assert route.call_count == 1 + assert response.choices[0].message.content == "Done." + assert "responses_api_version" not in request_body + + def test_azure_responses_sends_the_deployment_name_when_azure_ai_prefix_survives_provider_remap(): request = AzureOpenAIResponsesAPIConfig().transform_responses_api_request( model="azure_ai/gpt-5.4-nano", diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 9514d64615a..a445d951bc2 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -345,6 +345,80 @@ def test_azure_ai_grok_stop_parameter_handling(): assert "stop" in gpt_params, "GPT models should support stop parameter" +@pytest.mark.respx(assert_all_called=True) +def test_azure_ai_grok_bills_reasoning_tokens_reported_outside_completion_tokens( + respx_mock: respx.MockRouter, +) -> None: + api_url: Final = "https://example-resource.services.ai.azure.com/models/chat/completions" + route: Final = respx_mock.post(url__startswith=api_url).mock( + return_value=Response( + 200, + json={ + "id": "chatcmpl-grok", + "object": "chat.completion", + "created": 1700000000, + "model": "grok-4.7", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "42"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 14, + "completion_tokens": 10, + "total_tokens": 336, + "completion_tokens_details": {"reasoning_tokens": 312}, + }, + }, + ) + ) + + response: Final = litellm.completion( + model="azure_ai/grok-4.7", + messages=[{"role": "user", "content": "What is the meaning of life?"}], + api_base="https://example-resource.services.ai.azure.com/models", + api_key="azure-ai-test-key", + ) + model_info: Final = litellm.get_model_info(model="grok-4.7", custom_llm_provider="azure_ai") + expected_cost: Final = 14 * model_info["input_cost_per_token"] + 322 * model_info["output_cost_per_token"] + + assert route.call_count == 1 + assert response.usage.completion_tokens == 322 + assert response.usage.total_tokens == response.usage.prompt_tokens + response.usage.completion_tokens + assert response._hidden_params["response_cost"] == pytest.approx(expected_cost, rel=1e-9) + + +@pytest.mark.respx(assert_all_called=True) +def test_azure_ai_grok_stream_folds_reasoning_tokens_into_the_usage_chunk(respx_mock: respx.MockRouter) -> None: + api_url: Final = "https://example-resource.services.ai.azure.com/models/chat/completions" + stream_body: Final = ( + 'data: {"id":"chatcmpl-grok","object":"chat.completion.chunk","created":1700000000,"model":"grok-4.7",' + '"choices":[{"index":0,"delta":{"role":"assistant","content":"42"},"finish_reason":null}]}\n\n' + 'data: {"id":"chatcmpl-grok","object":"chat.completion.chunk","created":1700000000,"model":"grok-4.7",' + '"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n' + 'data: {"id":"chatcmpl-grok","object":"chat.completion.chunk","created":1700000000,"model":"grok-4.7",' + '"choices":[],"usage":{"prompt_tokens":14,"completion_tokens":10,"total_tokens":336,' + '"completion_tokens_details":{"reasoning_tokens":312}}}\n\n' + "data: [DONE]\n\n" + ) + route: Final = respx_mock.post(url__startswith=api_url).mock( + return_value=Response(200, text=stream_body, headers={"content-type": "text/event-stream"}) + ) + + chunks: Final = tuple( + litellm.completion( + model="azure_ai/grok-4.7", + messages=[{"role": "user", "content": "What is the meaning of life?"}], + api_base="https://example-resource.services.ai.azure.com/models", + api_key="azure-ai-test-key", + stream=True, + stream_options={"include_usage": True}, + ) + ) + usage_chunks: Final = [chunk.usage for chunk in chunks if getattr(chunk, "usage", None) is not None] + + assert route.call_count == 1 + assert len(usage_chunks) == 1 + assert usage_chunks[0].completion_tokens == 322 + assert usage_chunks[0].total_tokens == usage_chunks[0].prompt_tokens + usage_chunks[0].completion_tokens + + def test_azure_model_router_response_shows_actual_model(): """ Test that Azure Model Router returns the actual model used in the response, diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 0da28e7207e..f85e4db447b 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -1434,11 +1434,11 @@ class TestAzureResponsesAPIConfig: # Test with date-based version - should use openai/responses result_date = self.config.get_complete_url( api_base=base_url, - litellm_params={"api_version": "2025-01-01"}, + litellm_params={"api_version": "2025-04-01-preview"}, ) assert ( result_date - == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" + == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview" ) def test_azure_transform_then_normalize_strips_custom_tool_call_namespace(self): diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 9f9a6687210..660cc93874d 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -932,6 +932,26 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion(): assert fake_model not in litellm.models_by_provider["vertex_ai"] +def test_wildcard_lists_a_model_once_when_registered_both_bare_and_provider_prefixed(): + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + bare: Final = "gpt-dedupe-regression" + entry: Final = {"litellm_provider": "openai", "mode": "chat"} + try: + litellm.register_model({bare: entry, f"openai/{bare}": entry}, persist_across_reloads=False) + + listed: Final = get_known_models_from_wildcard("openai/*") + + assert listed.count(f"openai/{bare}") == 1 + finally: + litellm.model_cost.pop(bare, None) + litellm.model_cost.pop(f"openai/{bare}", None) + litellm.open_ai_chat_completion_models.discard(bare) + litellm.open_ai_chat_completion_models.discard(f"openai/{bare}") + litellm.add_known_models(model_cost_map={}) + + def test_get_complete_model_list_drops_no_default_models_sentinel(): from litellm.proxy.auth.model_checks import get_complete_model_list diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 253850bbd79..e11c5d6c2f3 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -5357,6 +5357,7 @@ class TestRouterPreRoutingSharedAliasName: "api_key": "sk-marker", "api_base": "https://marker.example/v1", "api_version": "2024-01-01", + "responses_api_version": "v1", }, } router = Router(model_list=[marker_with_connection_params, self._tier_entry()]) @@ -5372,6 +5373,7 @@ class TestRouterPreRoutingSharedAliasName: assert "api_base" not in request_kwargs assert "api_key" not in request_kwargs assert "api_version" not in request_kwargs + assert "responses_api_version" not in request_kwargs assert request_kwargs["drop_params"] is True @pytest.mark.asyncio diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index b12d3b34c8b..ea484c4a2f4 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -419,6 +419,24 @@ def test_azure_gpt_6_1_sol_data_zone_rows_price_requests_at_the_regional_premium assert total_cost(f"azure/{region}/gpt-6.1-sol") == pytest.approx(global_cost * premium) +AZURE_GPT_6_1_SOL_ROWS: Final = ( + "azure/gpt-6.1-sol", + "azure/gpt-6.1-sol-2026-09-29", + "azure/us/gpt-6.1-sol", + "azure/eu/gpt-6.1-sol", + "azure/apac/gpt-6.1-sol", + "azure_ai/gpt-6.1-sol", +) + + +@pytest.mark.parametrize("row", AZURE_GPT_6_1_SOL_ROWS) +def test_azure_gpt_6_1_sol_rows_accept_the_same_reasoning_efforts_as_the_openai_row(prices: dict, row: str): + openai_efforts: Final = resolve_supported_reasoning_efforts(prices["gpt-6.1-sol"], deployment_is_mapped=True) + assert openai_efforts + + assert resolve_supported_reasoning_efforts(prices[row], deployment_is_mapped=True) == openai_efforts + + # https://ai.google.dev/gemini-api/docs/models/deep-research-preview-04-2026 @pytest.mark.parametrize("model", ["gemini/deep-research-preview-04-2026", "gemini/deep-research-max-preview-04-2026"]) def test_gemini_deep_research_previews_accept_the_documented_input_window(prices: dict, model: str): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 75cec231f16..632ab020409 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2418,6 +2418,87 @@ def test_register_model_openrouter_without_slash(): assert "openai/gpt-4-turbo" in litellm.openrouter_models +def test_register_model_makes_the_model_resolvable_and_wildcard_visible_without_a_restart(): + model_name: Final = "my-tuned-gemini-regression" + try: + litellm.register_model( + { + model_name: { + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + } + }, + persist_across_reloads=False, + ) + + _, provider, _, _ = litellm.get_llm_provider(model=model_name) + + assert provider == "vertex_ai" + assert model_name in litellm.models_by_provider["vertex_ai"] + finally: + litellm.model_cost.pop(model_name, None) + litellm.vertex_language_models.discard(model_name) + litellm.models_by_provider["vertex_ai"].discard(model_name) + + +@pytest.mark.parametrize( + ("litellm_provider", "wildcard", "provider_set"), + [ + ("ai21", "ai21/*", "ai21_chat_models"), + ("vertex_ai-code-chat-models", "vertex_ai/*", "vertex_code_chat_models"), + ], +) +def test_registered_chat_model_lists_under_its_provider_wildcard( + litellm_provider: str, wildcard: str, provider_set: str +) -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + model_name: Final = f"chat-listing-regression-{litellm_provider}" + try: + litellm.register_model( + {model_name: {"litellm_provider": litellm_provider, "mode": "chat"}}, persist_across_reloads=False + ) + + listed: Final = get_known_models_from_wildcard(wildcard) + + assert f"{wildcard.removesuffix('*')}{model_name}" in listed + finally: + litellm.model_cost.pop(model_name, None) + getattr(litellm, provider_set).discard(model_name) + litellm.add_known_models(model_cost_map={}) + + +def test_priced_router_deployment_id_stays_out_of_wildcard_model_listing(): + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + deployment_id: Final = "priced-deployment-id-regression" + router: Final = litellm.Router( + model_list=[ + { + "model_name": "priced-gemini", + "litellm_params": { + "model": "gemini/gemini-2.5-flash", + "api_key": "fake-key", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + }, + "model_info": {"id": deployment_id, "litellm_provider": "gemini", "mode": "chat"}, + }, + {"model_name": "gemini/*", "litellm_params": {"model": "gemini/*", "api_key": "fake-key"}}, + ] + ) + + router.completion(model="priced-gemini", messages=[{"role": "user", "content": "hi"}], mock_response="ok") + listed: Final = get_known_models_from_wildcard("gemini/*") + + assert "gemini/gemini-2.5-flash" in listed + assert f"gemini/{deployment_id}" not in listed + assert deployment_id not in litellm.models_by_provider["gemini"] + assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.000001 + + def test_reasoning_content_preserved_in_text_completion_wrapper(): """Ensure reasoning_content is copied from delta to text_choices.""" chunk = ModelResponseStream( diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 122d811e491..aef43a15a72 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -65,6 +65,7 @@ CONNECTION_NAMES: Final = ( "api_key", "api_base", "api_version", + "responses_api_version", "region_name", "headers", "provider_specific_header", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 18ce90fb19c..7fae3437f22 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35325,6 +35325,8 @@ export interface components { regional_processing_uplift_multiplier_eu?: number | null; /** Regional Processing Uplift Multiplier Us */ regional_processing_uplift_multiplier_us?: number | null; + /** Responses Api Version */ + responses_api_version?: string | null; /** Rpm */ rpm?: number | null; /** S3 Access Key Id */ @@ -50550,6 +50552,8 @@ export interface components { regional_processing_uplift_multiplier_eu?: number | null; /** Regional Processing Uplift Multiplier Us */ regional_processing_uplift_multiplier_us?: number | null; + /** Responses Api Version */ + responses_api_version?: string | null; /** Rpm */ rpm?: number | null; /** S3 Access Key Id */