diff --git a/litellm/__init__.py b/litellm/__init__.py index 83e7642c39f..df52808643d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1177,7 +1177,9 @@ def _build_models_by_provider() -> dict: "edenai": edenai_models, "datarobot": datarobot_models, "vertex_ai": vertex_chat_models + | vertex_code_chat_models | vertex_text_models + | vertex_code_text_models | vertex_anthropic_models | vertex_vision_models | vertex_language_models @@ -1185,7 +1187,7 @@ def _build_models_by_provider() -> dict: | vertex_minimax_models | vertex_moonshot_models | vertex_zai_models, - "ai21": ai21_models, + "ai21": ai21_models | ai21_chat_models, "bedrock": bedrock_models | bedrock_converse_models, "petals": petals_models, "ollama": ollama_models, diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 04755f9e1e9..9dee3714c60 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -78,6 +78,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "github_copilot_auth_type", "github_copilot_user_session", "fireworks_forward_user_id", + "responses_api_version", PROVIDER_AFFINITY_HEADER_KWARG_KEY, } ) diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index 82150e415aa..a23fffb5744 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -95,7 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): status_code=400, message=( "Azure OpenAI does not support reasoning_effort='none' for this model. " - "Supported values are: 'low', 'medium', and 'high'. " "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n" "`litellm_settings:\n drop_params: true`\n" "Issue: https://github.com/BerriAI/litellm/issues/16704" diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 3564d7a1364..46106026920 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -2,6 +2,7 @@ import asyncio import hashlib import json import os +import re from collections.abc import Callable, Mapping from functools import lru_cache from types import MappingProxyType @@ -25,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.utils import add_path_to_api_base azure_ad_cache: Final = DualCache() +_LAST_OPENAI_PATH_SEGMENT: Final = re.compile(r"^(.*/openai)(?=/|$)") class _AzureAdTokenJson(TypedDict, total=False): @@ -798,6 +800,10 @@ class BaseAzureLLM(BaseOpenAILLM): ) -> dict[str, str]: # mutable-ok: mirrors override contract return cls._base_validate_azure_environment(headers, litellm_params) + @staticmethod + def resolve_azure_api_base(api_base: str | None) -> str | None: + return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") + @staticmethod def _get_base_azure_url( api_base: str | None, @@ -815,7 +821,7 @@ class BaseAzureLLM(BaseOpenAILLM): default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route. """ - api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") + api_base = BaseAzureLLM.resolve_azure_api_base(api_base) if api_base is None: raise ValueError( f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`" @@ -843,7 +849,7 @@ class BaseAzureLLM(BaseOpenAILLM): # ensure the request go to /openai/v1 and not just /openai if "/openai/v1" not in new_url: parsed_url: Final = httpx.URL(new_url) - new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1"))) + new_url = str(parsed_url.copy_with(path=_LAST_OPENAI_PATH_SEGMENT.sub(r"\1/v1", parsed_url.path, 1))) # Use the new query_params dictionary final_url: Final = httpx.URL(new_url).copy_with(params=query_params) diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 98b4241d7ab..75714d5253b 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -1,4 +1,7 @@ +import re +from collections.abc import Mapping from copy import deepcopy +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -6,6 +9,7 @@ from openai.types.responses import ResponseReasoningItem from pydantic import TypeAdapter from litellm._logging import verbose_logger +from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import BaseAzureLLM @@ -25,6 +29,34 @@ else: _DECODED_JSON: Final = TypeAdapter(object) +_DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}") + + +def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None: + value: Final = litellm_params.get(key) + return value if isinstance(value, str) and value else None + + +def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> str | None: + """ + Dated GA api_versions (e.g. 2024-10-21) predate the Responses API on Azure and 404 there, so a deployment + pinned to one for chat gets the default Responses version unless it sets responses_api_version. + """ + api_version: Final = _str_param(litellm_params, "api_version") + if api_base is not None and "api-version" in httpx.URL(api_base).params: + return api_version + responses_api_version: Final = _str_param(litellm_params, "responses_api_version") + if responses_api_version is not None: + return responses_api_version + if api_version is not None and _DATED_GA_API_VERSION.fullmatch(api_version): + return AZURE_DEFAULT_RESPONSES_API_VERSION + return api_version + + +def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)}) + + class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): # Parameters not supported by Azure Responses API AZURE_UNSUPPORTED_PARAMS = ["context_management"] @@ -170,11 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): - A complete URL string, e.g., "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview" """ - from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION - + resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base) return BaseAzureLLM.get_base_azure_url( - api_base=api_base, - litellm_params=litellm_params, + api_base=resolved_api_base, + litellm_params=_with_responses_api_version(resolved_api_base, litellm_params), route="/openai/responses", default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION, ) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index a8e6320bd1e..fc22e9a8b73 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -1,6 +1,7 @@ import copy import enum import re +from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Final, cast import httpx @@ -21,12 +22,12 @@ from litellm.llms.azure_ai.common_utils import ( from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error -from litellm.llms.openai.openai import OpenAIConfig +from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig from litellm.llms.xai.chat.transformation import XAIChatConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import ModelResponse, ProviderField +from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField from litellm.utils import add_path_to_api_base, supports_tool_choice if TYPE_CHECKING: @@ -45,6 +46,18 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ( ) +def _is_grok_model(model: str) -> bool: + return "grok" in model + + +class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator): + def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract + usage: Final = chunk.get("usage") + if _is_grok_model(str(chunk.get("model") or "")) and isinstance(usage, dict): + XAIChatConfig.fold_reasoning_tokens_into_completion(usage) + return super().chunk_parser(chunk) + + class AzureAIGPT5Config(OpenAIGPT5Config): @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -114,7 +127,7 @@ class AzureAIStudioConfig(OpenAIConfig): """ Check if the model supports stop tokens. """ - if "grok" in model: + if _is_grok_model(model): # Reuse Xai method for Grok model xai_config: Final = XAIChatConfig() return xai_config.supports_stop_reason(model) @@ -319,7 +332,7 @@ class AzureAIStudioConfig(OpenAIConfig): json_mode: bool | None = None, ) -> ModelResponse: model_response.model = f"azure_ai/{model}" - return super().transform_response( + response: Final = super().transform_response( model=model, raw_response=raw_response, model_response=model_response, @@ -332,6 +345,21 @@ class AzureAIStudioConfig(OpenAIConfig): api_key=api_key, json_mode=json_mode, ) + if _is_grok_model(model): + XAIChatConfig.fold_reasoning_tokens_into_completion(response) + return response + + def get_model_response_iterator( + self, + streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, + sync_stream: bool, + json_mode: bool | None = False, + ) -> "AzureAIChatCompletionResponseIterator": + return AzureAIChatCompletionResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) def should_retry_llm_api_inside_llm_translation_on_http_error( self, e: httpx.HTTPStatusError, litellm_params: dict diff --git a/litellm/main.py b/litellm/main.py index e6caa5c25b6..3646b48b8cc 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1212,13 +1212,18 @@ def _register_custom_pricing_for_request( litellm.register_model({shared_key: entry}, persist_across_reloads=False) return litellm.register_model( - { - deployment_id: entry, - shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry), - }, + {deployment_id: entry}, persist_across_reloads=False, warning_display_name=shared_key, custom_llm_provider=custom_llm_provider, + add_to_provider_model_sets=False, + ) + litellm.register_model( + {shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)}, + persist_across_reloads=False, + warning_display_name=shared_key, + custom_llm_provider=custom_llm_provider, + add_to_provider_model_sets=False, ) @@ -5785,6 +5790,7 @@ def completion( "github_copilot_user_session", PROVIDER_AFFINITY_HEADER_KWARG_KEY, "fireworks_forward_user_id", + "responses_api_version", ) if key in kwargs }, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e811045bf1a..47208d26e2a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4029,7 +4029,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_cache_breakpoint": true, @@ -8698,7 +8698,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8747,7 +8747,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8796,7 +8796,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8845,7 +8845,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8894,7 +8894,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index b70cdb08b96..8a7b6dab419 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -332,7 +332,7 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ else: model = f"{wildcard_provider_prefix}/{model}" suffix_appended_wildcard_models.append(model) - return suffix_appended_wildcard_models or [] + return list(dict.fromkeys(suffix_appended_wildcard_models)) def expand_wildcard_deployments_for_model_info( diff --git a/litellm/router.py b/litellm/router.py index d041f958e17..99adc6c5fa6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -501,7 +501,9 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _CallbackT = TypeVar("_CallbackT") -_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) +_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset( + {"model", "api_base", "api_key", "api_version", "responses_api_version"} +) _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" _CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") _CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600 @@ -10995,6 +10997,7 @@ class Router: persist_across_reloads=False, warning_display_name=model, custom_llm_provider=custom_llm_provider, + add_to_provider_model_sets=False, ) ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 8387afa22bb..174d07a2cea 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -67,6 +67,7 @@ class ProviderConnection: api_key: str | None = None api_base: str | None = None api_version: str | None = None + responses_api_version: str | None = None region_name: str | None = None headers: Mapping[str, str] | None = None provider_specific_header: "ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None" = None diff --git a/litellm/types/router.py b/litellm/types/router.py index 0ca39e2400d..8f3ba7c0a8d 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -496,6 +496,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): def __init__(self, /, **data: object) -> None: ... # kwargs-ok: provider parameters vary by deployment custom_llm_provider: str | None = None + responses_api_version: str | None = None tpm: int | None = None rpm: int | None = None itpm: int | None = None @@ -703,6 +704,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): api_key: str | None api_base: str | None api_version: str | None + responses_api_version: ReadOnly[str | None] timeout: float | str | httpx.Timeout | SDKTimeout | None # writable-ok: preserve timeout updates stream_timeout: float | str | None max_retries: int | None diff --git a/litellm/utils.py b/litellm/utils.py index 76ef50c3a2b..3b43c570466 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3445,6 +3445,7 @@ def register_model( persist_across_reloads: bool = True, warning_display_name: str | None = None, custom_llm_provider: str | None = None, + add_to_provider_model_sets: bool = True, ): """ Register new / Override existing models (and their pricing) to specific providers. @@ -3473,6 +3474,10 @@ def register_model( ``custom_llm_provider`` scopes the built-in cost-map match to entries for that provider, so a deployment id that happens to equal another provider's catalog key stays its own provider-less entry instead of merging into it. + + ``add_to_provider_model_sets`` adds the keys to the per-provider model sets + behind provider resolution and wildcard model listing. Pass False when the + keys are opaque ids rather than model names (e.g. router deployment ids). """ loaded_model_cost = {} @@ -3558,56 +3563,10 @@ def register_model( _invalidate_model_cost_lowercase_map() verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", model_cost_key, model_cost_key) - # add new model names to provider lists - if value.get("litellm_provider") == "openai": - if key not in litellm.open_ai_chat_completion_models: - litellm.open_ai_chat_completion_models.add(key) - elif value.get("litellm_provider") == "text-completion-openai": - if key not in litellm.open_ai_text_completion_models: - litellm.open_ai_text_completion_models.add(key) - elif value.get("litellm_provider") == "cohere": - if key not in litellm.cohere_models: - litellm.cohere_models.add(key) - elif value.get("litellm_provider") == "anthropic": - if key not in litellm.anthropic_models: - litellm.anthropic_models.add(key) - elif value.get("litellm_provider") == "openrouter": - split_string = key.split("/", 1) - if split_string[-1] not in litellm.openrouter_models: - litellm.openrouter_models.add(split_string[-1]) - elif value.get("litellm_provider") == "vercel_ai_gateway": - if key not in litellm.vercel_ai_gateway_models: - litellm.vercel_ai_gateway_models.add(key) - elif value.get("litellm_provider") == "edenai": - if key not in litellm.edenai_models: - litellm.edenai_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-text-models": - if key not in litellm.vertex_text_models: - litellm.vertex_text_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-code-text-models": - if key not in litellm.vertex_code_text_models: - litellm.vertex_code_text_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-chat-models": - if key not in litellm.vertex_chat_models: - litellm.vertex_chat_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-code-chat-models": - if key not in litellm.vertex_code_chat_models: - litellm.vertex_code_chat_models.add(key) - elif value.get("litellm_provider") == "ai21": - if key not in litellm.ai21_models: - litellm.ai21_models.add(key) - elif value.get("litellm_provider") == "nlp_cloud": - if key not in litellm.nlp_cloud_models: - litellm.nlp_cloud_models.add(key) - elif value.get("litellm_provider") == "aleph_alpha": - if key not in litellm.aleph_alpha_models: - litellm.aleph_alpha_models.add(key) - elif value.get("litellm_provider") == "bedrock": - if key not in litellm.bedrock_models: - litellm.bedrock_models.add(key) - elif value.get("litellm_provider") == "novita": - if key not in litellm.novita_models: - litellm.novita_models.add(key) + if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter": + litellm.openrouter_models.add(_key_str.split("/", 1)[-1]) + if add_to_provider_model_sets: + litellm.add_known_models(loaded_model_cost) return model_cost diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e811045bf1a..47208d26e2a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4029,7 +4029,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_cache_breakpoint": true, @@ -8698,7 +8698,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8747,7 +8747,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8796,7 +8796,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8845,7 +8845,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8894,7 +8894,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/tests/e2e/llm_translation/responses_helpers.py b/tests/e2e/llm_translation/responses_helpers.py index be0fac74bb6..29b923a7f34 100644 --- a/tests/e2e/llm_translation/responses_helpers.py +++ b/tests/e2e/llm_translation/responses_helpers.py @@ -8,10 +8,13 @@ AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" AZURE_OPENAI_API_VERSION: Final = "v1" -def azure_openai_params(api_version: str = AZURE_OPENAI_API_VERSION) -> LiteLLMParamsBody: +def azure_openai_params( + api_version: str = AZURE_OPENAI_API_VERSION, responses_api_version: str | None = None +) -> LiteLLMParamsBody: return LiteLLMParamsBody( model=AZURE_OPENAI_BACKEND, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY", api_version=api_version, + responses_api_version=responses_api_version, ) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index ee4699703b2..bb75569ad99 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -40,7 +40,7 @@ from provider_edge import LiveEdge, start_provider_edge from provider_edge_bedrock import ConverseBodyCapture, bedrock_signer from proxy_client import ProxyClient from pydantic import BaseModel, TypeAdapter -from responses_helpers import AZURE_OPENAI_BACKEND, azure_openai_params +from responses_helpers import AZURE_OPENAI_API_VERSION, AZURE_OPENAI_BACKEND, azure_openai_params from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -748,6 +748,70 @@ class TestResponses: f"/responses over azure openai preview returned no output text: {response.output!r}" ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=(AZURE_OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_azure_openai_dated_ga_chat_api_version_serves_responses( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register( + proxy, + resources, + azure_openai_params(api_version="2024-10-21"), + prefix="e2e-responses-azure-dated-chat", + ) + client: Final = sdk.openai(resources.key()) + + response: Final = client.responses.create( + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + extra_body=NO_PROXY_CACHE, + ) + assert response.output_text.strip(), ( + f"/responses over an azure deployment pinned to a dated GA chat api_version returned no output text: " + f"{response.output!r}" + ) + + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=(AZURE_OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_azure_openai_responses_api_version_overrides_a_pre_responses_api_version( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + # Azure rejects /openai/responses below api-version 2025-03-01-preview (probed live 2026-10-15), + # so this call only succeeds if the proxy sends responses_api_version instead of api_version. + model: Final = _register( + proxy, + resources, + azure_openai_params(api_version="2024-10-01-preview", responses_api_version=AZURE_OPENAI_API_VERSION), + prefix="e2e-responses-azure-override", + ) + client: Final = sdk.openai(resources.key()) + + response: Final = client.responses.create( + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + extra_body=NO_PROXY_CACHE, + ) + assert response.output_text.strip(), ( + f"/responses over an azure deployment with responses_api_version set returned no output text: " + f"{response.output!r}" + ) + @meta( Subject( domain=Domain.LLM_TRANSLATION, diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 08fc7a79df3..d7a10c6da1f 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1366,6 +1366,7 @@ class LiteLLMParamsBody(BaseModel): litellm_credential_name: str | None = None api_base: str | None = Field(default=None, repr=False) api_version: str | None = None + responses_api_version: str | None = None realtime_protocol: str | None = None allowed_openai_params: list[str] | None = None drop_params: bool | None = None diff --git a/tests/integration/cost_calculation/cases/misc.py b/tests/integration/cost_calculation/cases/misc.py index c9729353c8e..f98dda5fb0c 100644 --- a/tests/integration/cost_calculation/cases/misc.py +++ b/tests/integration/cost_calculation/cases/misc.py @@ -834,6 +834,61 @@ AZURE_AI_GPT_5_4_MINI_LATEST_STREAM: Final = CostTrackingTestCase( ) +# azure_ai/grok-4.6 (Grok on Azure AI Foundry reports reasoning_tokens outside completion_tokens, like xai) +AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION: Final = CostTrackingTestCase( + name="azure-ai-grok-4.6-reasoning_folded_into_completion", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="azure_ai/grok-4.6", + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "azure-ai-grok-4.6-reasoning"}], + "stream": False, + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "model": "grok-4.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2552, + "completion_tokens_details": {"reasoning_tokens": 300}, + }, + }, + ), + expected=ExactExpected( + spend=0.0108, input_cost=0.00368, output_cost=0.00712, prompt_tokens=1840, completion_tokens=712 + ), +) + +AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION_STREAM: Final = CostTrackingTestCase( + name="azure-ai-grok-4.6-reasoning_folded_into_completion-stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="azure_ai/grok-4.6", + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "azure-ai-grok-4.6-reasoning-stream"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + response=SseResponse( + content_type="text/event-stream", + frames=( + 'data: {"id":"chatcmpl-$REQUEST_ID","object":"chat.completion.chunk","created":1,"model":"grok-4.6","choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":null}]}\n\n', + 'data: {"id":"chatcmpl-$REQUEST_ID","object":"chat.completion.chunk","created":1,"model":"grok-4.6","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n', + 'data: {"id":"chatcmpl-$REQUEST_ID","object":"chat.completion.chunk","created":1,"model":"grok-4.6","choices":[],"usage":{"prompt_tokens":1840,"completion_tokens":412,"total_tokens":2552,"completion_tokens_details":{"reasoning_tokens":300}}}\n\n', + "data: [DONE]\n\n", + ), + ), + expected=ExactExpected( + spend=0.0108, input_cost=0.00368, output_cost=0.00712, prompt_tokens=1840, completion_tokens=712 + ), +) + + # azure_ai/claude-haiku-4-5 (Azure AI Foundry Anthropic, web search billed per server_tool_use.web_search_requests) AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH: Final = CostTrackingTestCase( name="azure-ai-claude-haiku-4-5-messages_web_search_two_requests", @@ -1978,6 +2033,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( BEDROCK_INVOKE_HAIKU_STREAM, AZURE_AI_GPT_5_4_MINI_LATEST, AZURE_AI_GPT_5_4_MINI_LATEST_STREAM, + AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION, + AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION_STREAM, AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH, AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH_STREAM, AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH_ONE_REQUEST, diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 5e2b99a113f..2ef42249a7e 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -764,6 +764,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token": 7.5e-07, "output_cost_per_token": 4.5e-06, }, + "azure_ai/grok-4.6": { + "litellm_provider": "azure_ai", + "mode": "chat", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + }, "azure_ai/claude-haiku-4-5": { "litellm_provider": "azure_ai", "mode": "chat", diff --git a/tests/integration/providers/test_azure_reasoning_effort_none_wire.py b/tests/integration/providers/test_azure_reasoning_effort_none_wire.py new file mode 100644 index 00000000000..c95e497c672 --- /dev/null +++ b/tests/integration/providers/test_azure_reasoning_effort_none_wire.py @@ -0,0 +1,58 @@ +from typing import Final + +from integration._support.client import Gateway +from integration._support.openai_wire import chat_reply +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_DEPLOYMENT: Final = "gpt-6.1-sol" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "no reasoning please"}] + + +def _reply(_request: Request) -> Reply: + return chat_reply("chatcmpl-effort-none", _DEPLOYMENT, "answered without reasoning", stream=False) + + +def test_reasoning_effort_none_is_rejected_by_litellm_before_reaching_azure(gateway: Gateway) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"azure/{_DEPLOYMENT}", + api_base=wire.url, + api_key="synthetic-azure-key", + api_version="2025-04-01-preview", + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _MESSAGES, "reasoning_effort": "none"}, + key=scenario.key(), + ) + + assert response.status_code == 400, response.text + assert "reasoning_effort='none'" in response.text, response.text + assert wire.drain() == () + + +def test_reasoning_effort_none_is_dropped_when_the_deployment_drops_params(gateway: Gateway) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"azure/{_DEPLOYMENT}", + api_base=wire.url, + api_key="synthetic-azure-key", + api_version="2025-04-01-preview", + drop_params=True, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _MESSAGES, "reasoning_effort": "none"}, + key=scenario.key(), + ) + + assert response.status_code == 200, response.text + assert "answered without reasoning" in response.text, response.text + received: Final = wire.drain() + assert [request.method for request in received] == ["POST"], [request.target for request in received] + body: Final = _JSON_OBJECT.validate_json(received[0].body) + assert "reasoning_effort" not in body and "reasoning" not in body, body diff --git a/tests/integration/providers/test_azure_responses_api_version_wire.py b/tests/integration/providers/test_azure_responses_api_version_wire.py new file mode 100644 index 00000000000..13c2f48dee2 --- /dev/null +++ b/tests/integration/providers/test_azure_responses_api_version_wire.py @@ -0,0 +1,108 @@ +from typing import Final +from urllib.parse import parse_qs, urlsplit + +from integration._support.client import Gateway, Scenario +from integration._support.openai_wire import chat_reply, responses_reply +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_DEPLOYMENT: Final = "gpt-5.4-nano" +_DATED_GA_API_VERSION: Final = "2024-10-21" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _reply(request: Request) -> Reply: + if urlsplit(request.target).path.endswith("/responses"): + return responses_reply("resp_azure_version", _DEPLOYMENT, "azure answered", stream=False) + return chat_reply("chatcmpl-azure-version", _DEPLOYMENT, "azure answered", stream=False) + + +def _dated_azure_model(scenario: Scenario, wire: Wire, model: str, responses_api_version: str | None = None) -> str: + if responses_api_version is None: + return scenario.model( + model=model, api_base=wire.url, api_key="synthetic-azure-key", api_version=_DATED_GA_API_VERSION + ) + return scenario.model( + model=model, + api_base=wire.url, + api_key="synthetic-azure-key", + api_version=_DATED_GA_API_VERSION, + responses_api_version=responses_api_version, + ) + + +def _only_post(wire: Wire) -> tuple[str, dict[str, str], dict[str, JsonValue]]: + posts: Final = tuple(request for request in wire.drain() if request.method == "POST") + assert len(posts) == 1, [request.target for request in posts] + target: Final = urlsplit(posts[0].target) + query: Final = {name: values[-1] for name, values in parse_qs(target.query).items()} + return target.path, query, _JSON_OBJECT.validate_json(posts[0].body) + + +def test_chat_keeps_the_dated_api_version_and_never_sends_responses_api_version(gateway: Gateway) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = _dated_azure_model(scenario, wire, f"azure/{_DEPLOYMENT}", responses_api_version="preview") + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "dated chat"}]}, + key=scenario.key(), + ) + + assert response.status_code == 200, response.text + path, query, body = _only_post(wire) + assert (path, query) == ( + f"/openai/deployments/{_DEPLOYMENT}/chat/completions", + {"api-version": _DATED_GA_API_VERSION}, + ) + assert body == { + "model": _DEPLOYMENT, + "messages": [{"role": "user", "content": "dated chat"}], + "stream": False, + }, body + + +def test_responses_on_a_dated_ga_deployment_reaches_the_v1_responses_route(gateway: Gateway) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = _dated_azure_model(scenario, wire, f"azure/{_DEPLOYMENT}") + response: Final = gateway.request( + "POST", "/v1/responses", {"model": model, "input": "dated responses"}, key=scenario.key() + ) + + assert response.status_code == 200, response.text + path, query, body = _only_post(wire) + assert (path, query) == ("/openai/v1/responses", {"api-version": "preview"}) + assert body["input"] == "dated responses", body + + +def test_responses_api_version_overrides_the_chat_api_version_and_stays_out_of_the_body(gateway: Gateway) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = _dated_azure_model( + scenario, wire, f"azure/{_DEPLOYMENT}", responses_api_version="2025-04-01-preview" + ) + response: Final = gateway.request( + "POST", "/v1/responses", {"model": model, "input": "override responses"}, key=scenario.key() + ) + + assert response.status_code == 200, response.text + path, query, body = _only_post(wire) + assert (path, query) == ("/openai/responses", {"api-version": "2025-04-01-preview"}) + assert "responses_api_version" not in body and "api_version" not in body, body + + +def test_chat_bridged_through_azure_responses_on_a_dated_ga_deployment_reaches_v1_responses( + gateway: Gateway, +) -> None: + with wire_server(_reply) as wire, gateway.scenario() as scenario: + model: Final = _dated_azure_model(scenario, wire, f"azure/responses/{_DEPLOYMENT}") + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "bridged chat"}]}, + key=scenario.key(), + ) + + assert response.status_code == 200, response.text + assert "azure answered" in response.text, response.text + path, query, _ = _only_post(wire) + assert (path, query) == ("/openai/v1/responses", {"api-version": "preview"}) diff --git a/tests/integration/routing/test_wildcard_model_listing.py b/tests/integration/routing/test_wildcard_model_listing.py new file mode 100644 index 00000000000..967a515ec08 --- /dev/null +++ b/tests/integration/routing/test_wildcard_model_listing.py @@ -0,0 +1,71 @@ +import uuid +from collections import Counter +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(180) + +_PRICED_DEPLOYMENT_ID: Final = f"priced-deployment-{uuid.uuid4().hex}" +_REGISTERED_AI21_CHAT_MODEL: Final = f"jamba-house-{uuid.uuid4().hex}" + + +@pytest.fixture(scope="module") +def candidate(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("wildcard-listing") + with gateway_from_environment() as base: + credentials: Final = {"api_base": f"{base.upstream_url}/v1", "api_key": "synthetic-listing-key"} + config: Final = { + **yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()), + "model_list": [ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", **credentials}}, + { + "model_name": "priced-chat", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + **credentials, + }, + "model_info": {"id": _PRICED_DEPLOYMENT_ID}, + }, + {"model_name": "ai21/*", "litellm_params": {"model": "ai21/*", **credentials}}, + { + "model_name": "house-jamba", + "litellm_params": {"model": f"ai21/{_REGISTERED_AI21_CHAT_MODEL}", **credentials}, + "model_info": {"litellm_provider": "ai21", "mode": "chat"}, + }, + ], + } + path: Final = directory / "wildcard-listing.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +def _listed_ids(gateway: Gateway) -> list[JsonValue]: + data: Final = gateway.get("/v1/models")["data"] + assert isinstance(data, list), data + return [entry["id"] for entry in data if isinstance(entry, dict)] + + +def test_wildcard_listing_after_a_priced_call_names_each_model_once_and_no_deployment_id(candidate: Gateway) -> None: + candidate.chat("priced-chat") + + listed: Final = _listed_ids(candidate) + + assert "openai/gpt-4o-mini" in listed and "priced-chat" in listed, listed + assert {model: count for model, count in Counter(listed).items() if count > 1} == {}, listed + assert [model for model in listed if _PRICED_DEPLOYMENT_ID in str(model)] == [], listed + + +def test_ai21_wildcard_listing_names_a_chat_model_registered_from_the_config(candidate: Gateway) -> None: + listed: Final = _listed_ids(candidate) + + assert f"ai21/{_REGISTERED_AI21_CHAT_MODEL}" in listed, listed diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index 9875cf439cf..f753fb6b8f9 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -753,3 +753,33 @@ def test_azure_deployment_id_selects_chat_completions_endpoint(respx_mock: MockR assert route.call_count == 1 assert route.calls[0].request.url.path == "/openai/deployments/deployment-gpt-4o/chat/completions" assert response.choices[0].message.content == "Hello" + + +@pytest.mark.respx(assert_all_called=True) +def test_responses_api_version_leaves_the_chat_route_and_body_untouched(respx_mock: MockRouter) -> None: + url: Final = "https://example-resource.openai.azure.com/openai/deployments/gpt-5.4-nano/chat/completions" + payload: Final = { + "id": "chatcmpl-dated", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-5.4-nano", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + } + route: Final = respx_mock.post(url__startswith=url).mock(return_value=httpx.Response(200, json=payload)) + messages: Final = [{"role": "user", "content": "Hello"}] + + response: Final = litellm.completion( + model="azure/gpt-5.4-nano", + messages=messages, + api_base="https://example-resource.openai.azure.com", + api_key="azure-test-key", + api_version="2024-12-01-preview", + responses_api_version="preview", + ) + request: Final = route.calls[0].request + + assert route.call_count == 1 + assert request.url.params["api-version"] == "2024-12-01-preview" + assert json.loads(request.content) == {"model": "gpt-5.4-nano", "messages": messages} + assert response.choices[0].message.content == "Hello" diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index fea5304a689..9b4c618c78b 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -1,11 +1,15 @@ import json +from collections.abc import Callable from copy import deepcopy from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +import respx import litellm +from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.azure.responses.o_series_transformation import ( @@ -282,16 +286,25 @@ class TestAzureResponsesAPIConfig: == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest" ) - # Test with date-based version - should use openai/responses - result_date = self.config.get_complete_url( + result_dated_preview = self.config.get_complete_url( api_base=base_url, - litellm_params={"api_version": "2025-01-01"}, + litellm_params={"api_version": "2025-04-01-preview"}, ) assert ( - result_date - == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" + result_dated_preview + == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview" ) + @pytest.mark.parametrize("api_version", ["2024-10-21", "2024-06-01", "2025-01-01"]) + def test_dated_ga_api_version_routes_responses_like_an_unset_one(self, api_version: str): + base_url: Final = "https://litellm8397336933.openai.azure.com" + + dated: Final = self.config.get_complete_url(api_base=base_url, litellm_params={"api_version": api_version}) + unset: Final = self.config.get_complete_url(api_base=base_url, litellm_params={}) + + assert dated == unset + assert httpx.URL(dated).params["api-version"] == AZURE_DEFAULT_RESPONSES_API_VERSION + def test_azure_get_complete_url_with_default_api_version(self): """Test Azure get_complete_url uses default API version when none is provided""" from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION @@ -320,6 +333,64 @@ class TestAzureResponsesAPIConfig: ) assert result_none_version == expected_url + @pytest.mark.parametrize( + ("api_version", "responses_api_version", "expected_path"), + [ + ("2024-10-21", "2025-04-01-preview", "/openai/responses?api-version=2025-04-01-preview"), + ("2025-04-01-preview", "v1", "/openai/v1/responses?api-version=v1"), + (None, "2025-04-01-preview", "/openai/responses?api-version=2025-04-01-preview"), + ], + ) + def test_responses_api_version_wins_over_api_version( + self, api_version: str | None, responses_api_version: str, expected_path: str + ): + base_url: Final = "https://litellm8397336933.openai.azure.com" + + url: Final = self.config.get_complete_url( + api_base=base_url, + litellm_params={"api_version": api_version, "responses_api_version": responses_api_version}, + ) + + assert url == f"{base_url}{expected_path}" + + @pytest.mark.parametrize("responses_api_version", [None, "v1"]) + def test_api_version_in_api_base_query_wins_over_a_dated_ga_api_version(self, responses_api_version: str | None): + api_base: Final = "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview" + + url: Final = self.config.get_complete_url( + api_base=api_base, + litellm_params={"api_version": "2024-10-21", "responses_api_version": responses_api_version}, + ) + + assert url == api_base + + def test_api_version_in_env_api_base_query_wins_over_a_dated_ga_api_version(self, monkeypatch: pytest.MonkeyPatch): + env_api_base: Final = ( + "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview" + ) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setenv("AZURE_API_BASE", env_api_base) + + url: Final = self.config.get_complete_url(api_base=None, litellm_params={"api_version": "2024-10-21"}) + + assert url == env_api_base + + @pytest.mark.parametrize( + ("api_base", "expected_path"), + [ + ("https://apim.example.com/openai-gw/team", "/openai-gw/team/openai/v1/responses"), + ("https://apim.example.com/openai/team-a", "/openai/team-a/openai/v1/responses"), + ], + ) + @pytest.mark.parametrize("litellm_params", [{"api_version": "2024-10-21"}, {"responses_api_version": "v1"}]) + def test_v1_route_only_rewrites_the_responses_route_segment_of_a_gateway_api_base( + self, api_base: str, expected_path: str, litellm_params: dict[str, str] + ): + url: Final = httpx.URL(self.config.get_complete_url(api_base=api_base, litellm_params=litellm_params)) + + assert url.host == "apim.example.com" + assert url.path == expected_path + def test_azure_cancel_response_api_request(self): """Test Azure cancel response API request transformation""" from litellm.types.router import GenericLiteLLMParams @@ -684,6 +755,207 @@ def test_azure_responses_gpt6_astra_rejects_temperature_while_reasoning(local_mo ) +_COMPLETED_RESPONSE: Final = { + "id": "resp_dated_chat_deployment", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "gpt-5.4-nano", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "truncation": None, + "user": None, +} + + +_AZURE_RESOURCE: Final = "https://example-resource.openai.azure.com" + + +def _unset_api_version_responses_url() -> httpx.URL: + return httpx.URL(AzureOpenAIResponsesAPIConfig().get_complete_url(api_base=_AZURE_RESOURCE, litellm_params={})) + + +def _azure_responses_create(api_version: str) -> str | None: + response: Final = litellm.responses( + model="azure/gpt-5.4-nano", + input="hi", + api_base=_AZURE_RESOURCE, + api_key="azure-test-key", + api_version=api_version, + ) + return response.output[0].content[0].text + + +def _azure_chat_bridged_to_responses(api_version: str) -> str | None: + response: Final = litellm.completion( + model="azure/responses/gpt-5.4-nano", + messages=[{"role": "user", "content": "hi"}], + api_base=_AZURE_RESOURCE, + api_key="azure-test-key", + api_version=api_version, + ) + return response.choices[0].message.content + + +@pytest.mark.parametrize("call_responses", [_azure_responses_create, _azure_chat_bridged_to_responses]) +def test_dated_ga_api_version_sends_responses_to_the_unset_api_version_url( + respx_mock: respx.MockRouter, call_responses: Callable[[str], str | None] +) -> None: + route: Final = respx_mock.post(host="example-resource.openai.azure.com").mock( + return_value=httpx.Response(200, json=_COMPLETED_RESPONSE) + ) + + text: Final = call_responses("2024-10-21") + + assert text == "Done." + assert route.call_count == 1 + assert route.calls[0].request.url == _unset_api_version_responses_url() + + +_LIFECYCLE_BODIES: Final = { + "get": _COMPLETED_RESPONSE, + "delete": {"id": "resp_dated", "object": "response", "deleted": True}, + "cancel": {**_COMPLETED_RESPONSE, "status": "cancelled"}, + "input_items": {"object": "list", "data": [], "first_id": None, "last_id": None, "has_more": False}, +} + + +def _call_lifecycle(operation: str) -> object: + kwargs: Final = { + "response_id": "resp_dated", + "custom_llm_provider": "azure", + "api_base": _AZURE_RESOURCE, + "api_key": "azure-test-key", + "api_version": "2024-10-21", + } + match operation: + case "get": + return litellm.get_responses(**kwargs) + case "delete": + return litellm.delete_responses(**kwargs) + case "cancel": + return litellm.cancel_responses(**kwargs) + case _: + return litellm.list_input_items(**kwargs) + + +@pytest.mark.parametrize( + ("operation", "path_suffix"), + [("get", ""), ("delete", ""), ("cancel", "/cancel"), ("input_items", "/input_items")], +) +def test_dated_ga_api_version_sends_response_lifecycle_calls_to_the_unset_api_version_route( + respx_mock: respx.MockRouter, operation: str, path_suffix: str +) -> None: + route: Final = respx_mock.route(host="example-resource.openai.azure.com").mock( + return_value=httpx.Response(200, json=_LIFECYCLE_BODIES[operation]) + ) + + _call_lifecycle(operation) + + unset_url: Final = _unset_api_version_responses_url() + sent_url: Final = route.calls[0].request.url + assert route.call_count == 1 + assert sent_url.path == f"{unset_url.path}/resp_dated{path_suffix}" + assert sent_url.params["api-version"] == unset_url.params["api-version"] + + +def test_chat_completion_keeps_the_dated_ga_api_version(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post( + f"{_AZURE_RESOURCE}/openai/deployments/gpt-5.4-nano/chat/completions", params={"api-version": "2024-10-21"} + ).mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-dated", + "object": "chat.completion", + "created": 1734366691, + "model": "gpt-5.4-nano", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Done."}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.completion( + model="azure/gpt-5.4-nano", + messages=[{"role": "user", "content": "hi"}], + api_base=_AZURE_RESOURCE, + api_key="azure-test-key", + api_version="2024-10-21", + ) + + assert response.choices[0].message.content == "Done." + assert route.call_count == 1 + + +@pytest.mark.respx(assert_all_called=True) +def test_responses_api_version_routes_a_dated_chat_deployment_to_its_own_responses_version( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post( + "https://example-resource.openai.azure.com/openai/responses", params={"api-version": "2025-04-01-preview"} + ).mock(return_value=httpx.Response(200, json=_COMPLETED_RESPONSE)) + + response: Final = litellm.responses( + model="azure/gpt-5.4-nano", + input="hi", + api_base="https://example-resource.openai.azure.com", + api_key="azure-test-key", + api_version="2024-10-21", + responses_api_version="2025-04-01-preview", + ) + request_body: Final = json.loads(route.calls[0].request.content) + + assert route.call_count == 1 + assert response.output[0].content[0].text == "Done." + assert request_body["model"] == "gpt-5.4-nano" + assert "responses_api_version" not in request_body + assert "api_version" not in request_body + + +@pytest.mark.respx(assert_all_called=True) +def test_responses_api_version_reaches_the_url_when_chat_completion_bridges_to_responses( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post( + "https://example-resource.openai.azure.com/openai/v1/responses", params={"api-version": "v1"} + ).mock(return_value=httpx.Response(200, json=_COMPLETED_RESPONSE)) + + response: Final = litellm.completion( + model="azure/responses/gpt-5.4-nano", + messages=[{"role": "user", "content": "hi"}], + api_base="https://example-resource.openai.azure.com", + api_key="azure-test-key", + api_version="2024-10-21", + responses_api_version="v1", + ) + request_body: Final = json.loads(route.calls[0].request.content) + + assert route.call_count == 1 + assert response.choices[0].message.content == "Done." + assert "responses_api_version" not in request_body + + def test_azure_responses_sends_the_deployment_name_when_azure_ai_prefix_survives_provider_remap(): request = AzureOpenAIResponsesAPIConfig().transform_responses_api_request( model="azure_ai/gpt-5.4-nano", diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 9514d64615a..a445d951bc2 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -345,6 +345,80 @@ def test_azure_ai_grok_stop_parameter_handling(): assert "stop" in gpt_params, "GPT models should support stop parameter" +@pytest.mark.respx(assert_all_called=True) +def test_azure_ai_grok_bills_reasoning_tokens_reported_outside_completion_tokens( + respx_mock: respx.MockRouter, +) -> None: + api_url: Final = "https://example-resource.services.ai.azure.com/models/chat/completions" + route: Final = respx_mock.post(url__startswith=api_url).mock( + return_value=Response( + 200, + json={ + "id": "chatcmpl-grok", + "object": "chat.completion", + "created": 1700000000, + "model": "grok-4.7", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "42"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 14, + "completion_tokens": 10, + "total_tokens": 336, + "completion_tokens_details": {"reasoning_tokens": 312}, + }, + }, + ) + ) + + response: Final = litellm.completion( + model="azure_ai/grok-4.7", + messages=[{"role": "user", "content": "What is the meaning of life?"}], + api_base="https://example-resource.services.ai.azure.com/models", + api_key="azure-ai-test-key", + ) + model_info: Final = litellm.get_model_info(model="grok-4.7", custom_llm_provider="azure_ai") + expected_cost: Final = 14 * model_info["input_cost_per_token"] + 322 * model_info["output_cost_per_token"] + + assert route.call_count == 1 + assert response.usage.completion_tokens == 322 + assert response.usage.total_tokens == response.usage.prompt_tokens + response.usage.completion_tokens + assert response._hidden_params["response_cost"] == pytest.approx(expected_cost, rel=1e-9) + + +@pytest.mark.respx(assert_all_called=True) +def test_azure_ai_grok_stream_folds_reasoning_tokens_into_the_usage_chunk(respx_mock: respx.MockRouter) -> None: + api_url: Final = "https://example-resource.services.ai.azure.com/models/chat/completions" + stream_body: Final = ( + 'data: {"id":"chatcmpl-grok","object":"chat.completion.chunk","created":1700000000,"model":"grok-4.7",' + '"choices":[{"index":0,"delta":{"role":"assistant","content":"42"},"finish_reason":null}]}\n\n' + 'data: {"id":"chatcmpl-grok","object":"chat.completion.chunk","created":1700000000,"model":"grok-4.7",' + '"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n' + 'data: {"id":"chatcmpl-grok","object":"chat.completion.chunk","created":1700000000,"model":"grok-4.7",' + '"choices":[],"usage":{"prompt_tokens":14,"completion_tokens":10,"total_tokens":336,' + '"completion_tokens_details":{"reasoning_tokens":312}}}\n\n' + "data: [DONE]\n\n" + ) + route: Final = respx_mock.post(url__startswith=api_url).mock( + return_value=Response(200, text=stream_body, headers={"content-type": "text/event-stream"}) + ) + + chunks: Final = tuple( + litellm.completion( + model="azure_ai/grok-4.7", + messages=[{"role": "user", "content": "What is the meaning of life?"}], + api_base="https://example-resource.services.ai.azure.com/models", + api_key="azure-ai-test-key", + stream=True, + stream_options={"include_usage": True}, + ) + ) + usage_chunks: Final = [chunk.usage for chunk in chunks if getattr(chunk, "usage", None) is not None] + + assert route.call_count == 1 + assert len(usage_chunks) == 1 + assert usage_chunks[0].completion_tokens == 322 + assert usage_chunks[0].total_tokens == usage_chunks[0].prompt_tokens + usage_chunks[0].completion_tokens + + def test_azure_model_router_response_shows_actual_model(): """ Test that Azure Model Router returns the actual model used in the response, diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 0da28e7207e..f85e4db447b 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -1434,11 +1434,11 @@ class TestAzureResponsesAPIConfig: # Test with date-based version - should use openai/responses result_date = self.config.get_complete_url( api_base=base_url, - litellm_params={"api_version": "2025-01-01"}, + litellm_params={"api_version": "2025-04-01-preview"}, ) assert ( result_date - == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" + == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview" ) def test_azure_transform_then_normalize_strips_custom_tool_call_namespace(self): diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 9f9a6687210..660cc93874d 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -932,6 +932,26 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion(): assert fake_model not in litellm.models_by_provider["vertex_ai"] +def test_wildcard_lists_a_model_once_when_registered_both_bare_and_provider_prefixed(): + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + bare: Final = "gpt-dedupe-regression" + entry: Final = {"litellm_provider": "openai", "mode": "chat"} + try: + litellm.register_model({bare: entry, f"openai/{bare}": entry}, persist_across_reloads=False) + + listed: Final = get_known_models_from_wildcard("openai/*") + + assert listed.count(f"openai/{bare}") == 1 + finally: + litellm.model_cost.pop(bare, None) + litellm.model_cost.pop(f"openai/{bare}", None) + litellm.open_ai_chat_completion_models.discard(bare) + litellm.open_ai_chat_completion_models.discard(f"openai/{bare}") + litellm.add_known_models(model_cost_map={}) + + def test_get_complete_model_list_drops_no_default_models_sentinel(): from litellm.proxy.auth.model_checks import get_complete_model_list diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 253850bbd79..e11c5d6c2f3 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -5357,6 +5357,7 @@ class TestRouterPreRoutingSharedAliasName: "api_key": "sk-marker", "api_base": "https://marker.example/v1", "api_version": "2024-01-01", + "responses_api_version": "v1", }, } router = Router(model_list=[marker_with_connection_params, self._tier_entry()]) @@ -5372,6 +5373,7 @@ class TestRouterPreRoutingSharedAliasName: assert "api_base" not in request_kwargs assert "api_key" not in request_kwargs assert "api_version" not in request_kwargs + assert "responses_api_version" not in request_kwargs assert request_kwargs["drop_params"] is True @pytest.mark.asyncio diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index b12d3b34c8b..ea484c4a2f4 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -419,6 +419,24 @@ def test_azure_gpt_6_1_sol_data_zone_rows_price_requests_at_the_regional_premium assert total_cost(f"azure/{region}/gpt-6.1-sol") == pytest.approx(global_cost * premium) +AZURE_GPT_6_1_SOL_ROWS: Final = ( + "azure/gpt-6.1-sol", + "azure/gpt-6.1-sol-2026-09-29", + "azure/us/gpt-6.1-sol", + "azure/eu/gpt-6.1-sol", + "azure/apac/gpt-6.1-sol", + "azure_ai/gpt-6.1-sol", +) + + +@pytest.mark.parametrize("row", AZURE_GPT_6_1_SOL_ROWS) +def test_azure_gpt_6_1_sol_rows_accept_the_same_reasoning_efforts_as_the_openai_row(prices: dict, row: str): + openai_efforts: Final = resolve_supported_reasoning_efforts(prices["gpt-6.1-sol"], deployment_is_mapped=True) + assert openai_efforts + + assert resolve_supported_reasoning_efforts(prices[row], deployment_is_mapped=True) == openai_efforts + + # https://ai.google.dev/gemini-api/docs/models/deep-research-preview-04-2026 @pytest.mark.parametrize("model", ["gemini/deep-research-preview-04-2026", "gemini/deep-research-max-preview-04-2026"]) def test_gemini_deep_research_previews_accept_the_documented_input_window(prices: dict, model: str): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 75cec231f16..632ab020409 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2418,6 +2418,87 @@ def test_register_model_openrouter_without_slash(): assert "openai/gpt-4-turbo" in litellm.openrouter_models +def test_register_model_makes_the_model_resolvable_and_wildcard_visible_without_a_restart(): + model_name: Final = "my-tuned-gemini-regression" + try: + litellm.register_model( + { + model_name: { + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + } + }, + persist_across_reloads=False, + ) + + _, provider, _, _ = litellm.get_llm_provider(model=model_name) + + assert provider == "vertex_ai" + assert model_name in litellm.models_by_provider["vertex_ai"] + finally: + litellm.model_cost.pop(model_name, None) + litellm.vertex_language_models.discard(model_name) + litellm.models_by_provider["vertex_ai"].discard(model_name) + + +@pytest.mark.parametrize( + ("litellm_provider", "wildcard", "provider_set"), + [ + ("ai21", "ai21/*", "ai21_chat_models"), + ("vertex_ai-code-chat-models", "vertex_ai/*", "vertex_code_chat_models"), + ], +) +def test_registered_chat_model_lists_under_its_provider_wildcard( + litellm_provider: str, wildcard: str, provider_set: str +) -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + model_name: Final = f"chat-listing-regression-{litellm_provider}" + try: + litellm.register_model( + {model_name: {"litellm_provider": litellm_provider, "mode": "chat"}}, persist_across_reloads=False + ) + + listed: Final = get_known_models_from_wildcard(wildcard) + + assert f"{wildcard.removesuffix('*')}{model_name}" in listed + finally: + litellm.model_cost.pop(model_name, None) + getattr(litellm, provider_set).discard(model_name) + litellm.add_known_models(model_cost_map={}) + + +def test_priced_router_deployment_id_stays_out_of_wildcard_model_listing(): + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + deployment_id: Final = "priced-deployment-id-regression" + router: Final = litellm.Router( + model_list=[ + { + "model_name": "priced-gemini", + "litellm_params": { + "model": "gemini/gemini-2.5-flash", + "api_key": "fake-key", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + }, + "model_info": {"id": deployment_id, "litellm_provider": "gemini", "mode": "chat"}, + }, + {"model_name": "gemini/*", "litellm_params": {"model": "gemini/*", "api_key": "fake-key"}}, + ] + ) + + router.completion(model="priced-gemini", messages=[{"role": "user", "content": "hi"}], mock_response="ok") + listed: Final = get_known_models_from_wildcard("gemini/*") + + assert "gemini/gemini-2.5-flash" in listed + assert f"gemini/{deployment_id}" not in listed + assert deployment_id not in litellm.models_by_provider["gemini"] + assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.000001 + + def test_reasoning_content_preserved_in_text_completion_wrapper(): """Ensure reasoning_content is copied from delta to text_choices.""" chunk = ModelResponseStream( diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 122d811e491..aef43a15a72 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -65,6 +65,7 @@ CONNECTION_NAMES: Final = ( "api_key", "api_base", "api_version", + "responses_api_version", "region_name", "headers", "provider_specific_header", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 18ce90fb19c..7fae3437f22 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35325,6 +35325,8 @@ export interface components { regional_processing_uplift_multiplier_eu?: number | null; /** Regional Processing Uplift Multiplier Us */ regional_processing_uplift_multiplier_us?: number | null; + /** Responses Api Version */ + responses_api_version?: string | null; /** Rpm */ rpm?: number | null; /** S3 Access Key Id */ @@ -50550,6 +50552,8 @@ export interface components { regional_processing_uplift_multiplier_eu?: number | null; /** Regional Processing Uplift Multiplier Us */ regional_processing_uplift_multiplier_us?: number | null; + /** Responses Api Version */ + responses_api_version?: string | null; /** Rpm */ rpm?: number | null; /** S3 Access Key Id */