mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-17 04:34:25 +00:00
fix(providers): route Azure Responses past dated api versions, count Grok reasoning tokens on azure_ai, refresh provider sets after register_model, fix gpt-6.1-sol Azure rows (#46377)
* fix(azure): add responses_api_version so the Responses route stops using the deployment chat api_version * fix(azure_ai): fold Grok reasoning tokens into completion_tokens like the xai provider * fix(utils): refresh provider model sets after register_model * fix(cost-map): align azure gpt-6.1-sol reasoning effort flags with the openai row * test(e2e): cover Azure Responses on a deployment pinned to a dated chat api_version * fix(utils): keep router deployment ids out of provider model sets register_model now refreshes the provider sets, so the per-request and router registrations keyed by a deployment id put those opaque ids into wildcard model listings. Those callers now pass add_to_provider_model_sets=False. * fix(azure): carry responses_api_version through the chat to Responses bridge completion() dropped responses_api_version from the litellm_params it hands the Responses bridge, so azure/responses/<model> on a dated chat api_version still built the 404ing /openai/responses URL. Also cover the auto-router marker never forwarding the new param. * fix(azure): stop listing a wrong set of reasoning efforts when none is rejected The message said only low, medium and high are supported, which is wrong for models that also take xhigh such as gpt-6.1-sol. * test(azure): use the latest grok and gpt-5.4 rows in the new azure tests * fix(proxy): list each wildcard-expanded model once A provider set can hold a model both bare and provider-prefixed, which now happens after any priced request registers its openai/<model> key, so the openai/* listing showed that model twice. * fix(azure): route Responses past dated GA api_versions by default A deployment pinned to a dated GA api_version such as 2024-10-21 for chat now sends Responses calls (create, get, delete, cancel, input_items and the azure/responses/ bridge) to the default Responses version, because those dated versions have no Responses route on Azure and 404. responses_api_version still overrides it, -preview, v1, latest and preview versions are kept, an api-version in the api_base query still wins, and chat keeps api_version as before * chore(azure_ai): mark the grok chunk_parser dict as an override contract * fix(azure): rewrite only the route's /openai segment when moving a call to v1 The v1 rewrite in the Azure URL builder replaced every "/openai" substring in the path, so a gateway api_base like https://apim.example.com/openai-gw/team turned into /openai/v1-gw/team/openai/v1/responses. Dated GA Responses calls now land on the v1 route by default, which made this reachable for them too, so the rewrite now only touches the last /openai path segment. The Responses version resolver no longer duplicates the builder's default for an unset api_version. Tests cover the gateway api_base case, the env AZURE_API_BASE query-string precedence, and the e2e suite now has one test for the dated GA default and one where only responses_api_version can make the call work * test(integration): cover Azure Responses api_version, Grok reasoning spend, wildcard listing and reasoning_effort none Wire tests pin the Azure route and query for chat, Responses and the azure/responses bridge on a dated GA api_version, and check responses_api_version never reaches the request body. Cost cases check azure_ai Grok folds reasoning tokens into completion_tokens and spend, streaming and not. A config-driven proxy checks /v1/models next to an openai/* wildcard lists each id once and never a deployment id. Azure gpt-6.1-sol rejects reasoning_effort none before reaching upstream and drops it with drop_params * fix(utils): list ai21 chat and vertex code models under their provider wildcard register_model now files ai21 chat models into ai21_chat_models, which models_by_provider never read, so a registered ai21 chat model dropped out of ai21/* listings. models_by_provider now reads ai21_chat_models and the vertex code chat and code text sets too. * fix(main): keep caller-chosen model names out of provider wildcard listings --------- Co-authored-by: yuneng-jiang <yuneng@berri.ai>
This commit is contained in:
parent
0403f8953e
commit
9cf09c18fe
32 changed files with 997 additions and 89 deletions
|
|
@ -1177,7 +1177,9 @@ def _build_models_by_provider() -> dict:
|
|||
"edenai": edenai_models,
|
||||
"datarobot": datarobot_models,
|
||||
"vertex_ai": vertex_chat_models
|
||||
| vertex_code_chat_models
|
||||
| vertex_text_models
|
||||
| vertex_code_text_models
|
||||
| vertex_anthropic_models
|
||||
| vertex_vision_models
|
||||
| vertex_language_models
|
||||
|
|
@ -1185,7 +1187,7 @@ def _build_models_by_provider() -> dict:
|
|||
| vertex_minimax_models
|
||||
| vertex_moonshot_models
|
||||
| vertex_zai_models,
|
||||
"ai21": ai21_models,
|
||||
"ai21": ai21_models | ai21_chat_models,
|
||||
"bedrock": bedrock_models | bedrock_converse_models,
|
||||
"petals": petals_models,
|
||||
"ollama": ollama_models,
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"github_copilot_auth_type",
|
||||
"github_copilot_user_session",
|
||||
"fireworks_forward_user_id",
|
||||
"responses_api_version",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -95,7 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
|
|||
status_code=400,
|
||||
message=(
|
||||
"Azure OpenAI does not support reasoning_effort='none' for this model. "
|
||||
"Supported values are: 'low', 'medium', and 'high'. "
|
||||
"To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n"
|
||||
"`litellm_settings:\n drop_params: true`\n"
|
||||
"Issue: https://github.com/BerriAI/litellm/issues/16704"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import asyncio
|
|||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Mapping
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
|
|
@ -25,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from litellm.utils import add_path_to_api_base
|
||||
|
||||
azure_ad_cache: Final = DualCache()
|
||||
_LAST_OPENAI_PATH_SEGMENT: Final = re.compile(r"^(.*/openai)(?=/|$)")
|
||||
|
||||
|
||||
class _AzureAdTokenJson(TypedDict, total=False):
|
||||
|
|
@ -798,6 +800,10 @@ class BaseAzureLLM(BaseOpenAILLM):
|
|||
) -> dict[str, str]: # mutable-ok: mirrors override contract
|
||||
return cls._base_validate_azure_environment(headers, litellm_params)
|
||||
|
||||
@staticmethod
|
||||
def resolve_azure_api_base(api_base: str | None) -> str | None:
|
||||
return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
|
||||
|
||||
@staticmethod
|
||||
def _get_base_azure_url(
|
||||
api_base: str | None,
|
||||
|
|
@ -815,7 +821,7 @@ class BaseAzureLLM(BaseOpenAILLM):
|
|||
default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route.
|
||||
"""
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
|
||||
api_base = BaseAzureLLM.resolve_azure_api_base(api_base)
|
||||
if api_base is None:
|
||||
raise ValueError(
|
||||
f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`"
|
||||
|
|
@ -843,7 +849,7 @@ class BaseAzureLLM(BaseOpenAILLM):
|
|||
# ensure the request go to /openai/v1 and not just /openai
|
||||
if "/openai/v1" not in new_url:
|
||||
parsed_url: Final = httpx.URL(new_url)
|
||||
new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1")))
|
||||
new_url = str(parsed_url.copy_with(path=_LAST_OPENAI_PATH_SEGMENT.sub(r"\1/v1", parsed_url.path, 1)))
|
||||
|
||||
# Use the new query_params dictionary
|
||||
final_url: Final = httpx.URL(new_url).copy_with(params=query_params)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
import re
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
|
||||
import httpx
|
||||
|
|
@ -6,6 +9,7 @@ from openai.types.responses import ResponseReasoningItem
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
|
||||
from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
|
|
@ -25,6 +29,34 @@ else:
|
|||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
_DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
|
||||
|
||||
|
||||
def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None:
|
||||
value: Final = litellm_params.get(key)
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
|
||||
def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> str | None:
|
||||
"""
|
||||
Dated GA api_versions (e.g. 2024-10-21) predate the Responses API on Azure and 404 there, so a deployment
|
||||
pinned to one for chat gets the default Responses version unless it sets responses_api_version.
|
||||
"""
|
||||
api_version: Final = _str_param(litellm_params, "api_version")
|
||||
if api_base is not None and "api-version" in httpx.URL(api_base).params:
|
||||
return api_version
|
||||
responses_api_version: Final = _str_param(litellm_params, "responses_api_version")
|
||||
if responses_api_version is not None:
|
||||
return responses_api_version
|
||||
if api_version is not None and _DATED_GA_API_VERSION.fullmatch(api_version):
|
||||
return AZURE_DEFAULT_RESPONSES_API_VERSION
|
||||
return api_version
|
||||
|
||||
|
||||
def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
|
||||
|
||||
|
||||
class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
||||
# Parameters not supported by Azure Responses API
|
||||
AZURE_UNSUPPORTED_PARAMS = ["context_management"]
|
||||
|
|
@ -170,11 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
- A complete URL string, e.g.,
|
||||
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
|
||||
"""
|
||||
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
|
||||
|
||||
resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
|
||||
return BaseAzureLLM.get_base_azure_url(
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
api_base=resolved_api_base,
|
||||
litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
|
||||
route="/openai/responses",
|
||||
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import copy
|
||||
import enum
|
||||
import re
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
import httpx
|
||||
|
|
@ -21,12 +22,12 @@ from litellm.llms.azure_ai.common_utils import (
|
|||
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
|
||||
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
|
||||
from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
|
||||
from litellm.llms.openai.openai import OpenAIConfig
|
||||
from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
|
||||
from litellm.llms.xai.chat.transformation import XAIChatConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ModelResponse, ProviderField
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
|
||||
from litellm.utils import add_path_to_api_base, supports_tool_choice
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -45,6 +46,18 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
|
|||
)
|
||||
|
||||
|
||||
def _is_grok_model(model: str) -> bool:
|
||||
return "grok" in model
|
||||
|
||||
|
||||
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
|
||||
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
|
||||
usage: Final = chunk.get("usage")
|
||||
if _is_grok_model(str(chunk.get("model") or "")) and isinstance(usage, dict):
|
||||
XAIChatConfig.fold_reasoning_tokens_into_completion(usage)
|
||||
return super().chunk_parser(chunk)
|
||||
|
||||
|
||||
class AzureAIGPT5Config(OpenAIGPT5Config):
|
||||
@classmethod
|
||||
def _model_map_lookup_name(cls, model: str) -> str:
|
||||
|
|
@ -114,7 +127,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
"""
|
||||
Check if the model supports stop tokens.
|
||||
"""
|
||||
if "grok" in model:
|
||||
if _is_grok_model(model):
|
||||
# Reuse Xai method for Grok model
|
||||
xai_config: Final = XAIChatConfig()
|
||||
return xai_config.supports_stop_reason(model)
|
||||
|
|
@ -319,7 +332,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
model_response.model = f"azure_ai/{model}"
|
||||
return super().transform_response(
|
||||
response: Final = super().transform_response(
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
|
|
@ -332,6 +345,21 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
api_key=api_key,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
if _is_grok_model(model):
|
||||
XAIChatConfig.fold_reasoning_tokens_into_completion(response)
|
||||
return response
|
||||
|
||||
def get_model_response_iterator(
|
||||
self,
|
||||
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
|
||||
sync_stream: bool,
|
||||
json_mode: bool | None = False,
|
||||
) -> "AzureAIChatCompletionResponseIterator":
|
||||
return AzureAIChatCompletionResponseIterator(
|
||||
streaming_response=streaming_response,
|
||||
sync_stream=sync_stream,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
|
||||
def should_retry_llm_api_inside_llm_translation_on_http_error(
|
||||
self, e: httpx.HTTPStatusError, litellm_params: dict
|
||||
|
|
|
|||
|
|
@ -1212,13 +1212,18 @@ def _register_custom_pricing_for_request(
|
|||
litellm.register_model({shared_key: entry}, persist_across_reloads=False)
|
||||
return
|
||||
litellm.register_model(
|
||||
{
|
||||
deployment_id: entry,
|
||||
shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry),
|
||||
},
|
||||
{deployment_id: entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
litellm.register_model(
|
||||
{shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -5785,6 +5790,7 @@ def completion(
|
|||
"github_copilot_user_session",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
"fireworks_forward_user_id",
|
||||
"responses_api_version",
|
||||
)
|
||||
if key in kwargs
|
||||
},
|
||||
|
|
|
|||
|
|
@ -4029,7 +4029,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
|
|
@ -8698,7 +8698,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -8747,7 +8747,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -8796,7 +8796,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -8845,7 +8845,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -8894,7 +8894,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
|
|||
|
|
@ -332,7 +332,7 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_
|
|||
else:
|
||||
model = f"{wildcard_provider_prefix}/{model}"
|
||||
suffix_appended_wildcard_models.append(model)
|
||||
return suffix_appended_wildcard_models or []
|
||||
return list(dict.fromkeys(suffix_appended_wildcard_models))
|
||||
|
||||
|
||||
def expand_wildcard_deployments_for_model_info(
|
||||
|
|
|
|||
|
|
@ -501,7 +501,9 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None
|
|||
_PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
|
||||
_CallbackT = TypeVar("_CallbackT")
|
||||
|
||||
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"})
|
||||
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset(
|
||||
{"model", "api_base", "api_key", "api_version", "responses_api_version"}
|
||||
)
|
||||
_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params"
|
||||
_CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
|
||||
_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600
|
||||
|
|
@ -10995,6 +10997,7 @@ class Router:
|
|||
persist_across_reloads=False,
|
||||
warning_display_name=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
|
||||
## OLD MODEL REGISTRATION ## Kept to prevent breaking changes
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ class ProviderConnection:
|
|||
api_key: str | None = None
|
||||
api_base: str | None = None
|
||||
api_version: str | None = None
|
||||
responses_api_version: str | None = None
|
||||
region_name: str | None = None
|
||||
headers: Mapping[str, str] | None = None
|
||||
provider_specific_header: "ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None" = None
|
||||
|
|
|
|||
|
|
@ -496,6 +496,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
|
|||
def __init__(self, /, **data: object) -> None: ... # kwargs-ok: provider parameters vary by deployment
|
||||
|
||||
custom_llm_provider: str | None = None
|
||||
responses_api_version: str | None = None
|
||||
tpm: int | None = None
|
||||
rpm: int | None = None
|
||||
itpm: int | None = None
|
||||
|
|
@ -703,6 +704,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
|
|||
api_key: str | None
|
||||
api_base: str | None
|
||||
api_version: str | None
|
||||
responses_api_version: ReadOnly[str | None]
|
||||
timeout: float | str | httpx.Timeout | SDKTimeout | None # writable-ok: preserve timeout updates
|
||||
stream_timeout: float | str | None
|
||||
max_retries: int | None
|
||||
|
|
|
|||
|
|
@ -3445,6 +3445,7 @@ def register_model(
|
|||
persist_across_reloads: bool = True,
|
||||
warning_display_name: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
add_to_provider_model_sets: bool = True,
|
||||
):
|
||||
"""
|
||||
Register new / Override existing models (and their pricing) to specific providers.
|
||||
|
|
@ -3473,6 +3474,10 @@ def register_model(
|
|||
``custom_llm_provider`` scopes the built-in cost-map match to entries for
|
||||
that provider, so a deployment id that happens to equal another provider's
|
||||
catalog key stays its own provider-less entry instead of merging into it.
|
||||
|
||||
``add_to_provider_model_sets`` adds the keys to the per-provider model sets
|
||||
behind provider resolution and wildcard model listing. Pass False when the
|
||||
keys are opaque ids rather than model names (e.g. router deployment ids).
|
||||
"""
|
||||
|
||||
loaded_model_cost = {}
|
||||
|
|
@ -3558,56 +3563,10 @@ def register_model(
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", model_cost_key, model_cost_key)
|
||||
# add new model names to provider lists
|
||||
if value.get("litellm_provider") == "openai":
|
||||
if key not in litellm.open_ai_chat_completion_models:
|
||||
litellm.open_ai_chat_completion_models.add(key)
|
||||
elif value.get("litellm_provider") == "text-completion-openai":
|
||||
if key not in litellm.open_ai_text_completion_models:
|
||||
litellm.open_ai_text_completion_models.add(key)
|
||||
elif value.get("litellm_provider") == "cohere":
|
||||
if key not in litellm.cohere_models:
|
||||
litellm.cohere_models.add(key)
|
||||
elif value.get("litellm_provider") == "anthropic":
|
||||
if key not in litellm.anthropic_models:
|
||||
litellm.anthropic_models.add(key)
|
||||
elif value.get("litellm_provider") == "openrouter":
|
||||
split_string = key.split("/", 1)
|
||||
if split_string[-1] not in litellm.openrouter_models:
|
||||
litellm.openrouter_models.add(split_string[-1])
|
||||
elif value.get("litellm_provider") == "vercel_ai_gateway":
|
||||
if key not in litellm.vercel_ai_gateway_models:
|
||||
litellm.vercel_ai_gateway_models.add(key)
|
||||
elif value.get("litellm_provider") == "edenai":
|
||||
if key not in litellm.edenai_models:
|
||||
litellm.edenai_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-text-models":
|
||||
if key not in litellm.vertex_text_models:
|
||||
litellm.vertex_text_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-code-text-models":
|
||||
if key not in litellm.vertex_code_text_models:
|
||||
litellm.vertex_code_text_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-chat-models":
|
||||
if key not in litellm.vertex_chat_models:
|
||||
litellm.vertex_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
|
||||
if key not in litellm.vertex_code_chat_models:
|
||||
litellm.vertex_code_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "ai21":
|
||||
if key not in litellm.ai21_models:
|
||||
litellm.ai21_models.add(key)
|
||||
elif value.get("litellm_provider") == "nlp_cloud":
|
||||
if key not in litellm.nlp_cloud_models:
|
||||
litellm.nlp_cloud_models.add(key)
|
||||
elif value.get("litellm_provider") == "aleph_alpha":
|
||||
if key not in litellm.aleph_alpha_models:
|
||||
litellm.aleph_alpha_models.add(key)
|
||||
elif value.get("litellm_provider") == "bedrock":
|
||||
if key not in litellm.bedrock_models:
|
||||
litellm.bedrock_models.add(key)
|
||||
elif value.get("litellm_provider") == "novita":
|
||||
if key not in litellm.novita_models:
|
||||
litellm.novita_models.add(key)
|
||||
if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter":
|
||||
litellm.openrouter_models.add(_key_str.split("/", 1)[-1])
|
||||
if add_to_provider_model_sets:
|
||||
litellm.add_known_models(loaded_model_cost)
|
||||
return model_cost
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4029,7 +4029,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
|
|
@ -8698,7 +8698,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -8747,7 +8747,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -8796,7 +8796,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -8845,7 +8845,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -8894,7 +8894,7 @@
|
|||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
|
|||
|
|
@ -8,10 +8,13 @@ AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
|
|||
AZURE_OPENAI_API_VERSION: Final = "v1"
|
||||
|
||||
|
||||
def azure_openai_params(api_version: str = AZURE_OPENAI_API_VERSION) -> LiteLLMParamsBody:
|
||||
def azure_openai_params(
|
||||
api_version: str = AZURE_OPENAI_API_VERSION, responses_api_version: str | None = None
|
||||
) -> LiteLLMParamsBody:
|
||||
return LiteLLMParamsBody(
|
||||
model=AZURE_OPENAI_BACKEND,
|
||||
api_base="os.environ/AZURE_API_BASE",
|
||||
api_key="os.environ/AZURE_API_KEY",
|
||||
api_version=api_version,
|
||||
responses_api_version=responses_api_version,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ from provider_edge import LiveEdge, start_provider_edge
|
|||
from provider_edge_bedrock import ConverseBodyCapture, bedrock_signer
|
||||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from responses_helpers import AZURE_OPENAI_BACKEND, azure_openai_params
|
||||
from responses_helpers import AZURE_OPENAI_API_VERSION, AZURE_OPENAI_BACKEND, azure_openai_params
|
||||
from sdk_clients import NO_PROXY_CACHE, SdkClients
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
|
@ -748,6 +748,70 @@ class TestResponses:
|
|||
f"/responses over azure openai preview returned no output text: {response.output!r}"
|
||||
)
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.RESPONSES,
|
||||
providers=(Provider.AZURE,),
|
||||
models=(AZURE_OPENAI_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_responses_azure_openai_dated_ga_chat_api_version_serves_responses(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register(
|
||||
proxy,
|
||||
resources,
|
||||
azure_openai_params(api_version="2024-10-21"),
|
||||
prefix="e2e-responses-azure-dated-chat",
|
||||
)
|
||||
client: Final = sdk.openai(resources.key())
|
||||
|
||||
response: Final = client.responses.create(
|
||||
model=model,
|
||||
input="reply with one word",
|
||||
instructions=INSTRUCTIONS,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
assert response.output_text.strip(), (
|
||||
f"/responses over an azure deployment pinned to a dated GA chat api_version returned no output text: "
|
||||
f"{response.output!r}"
|
||||
)
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.RESPONSES,
|
||||
providers=(Provider.AZURE,),
|
||||
models=(AZURE_OPENAI_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_responses_azure_openai_responses_api_version_overrides_a_pre_responses_api_version(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
# Azure rejects /openai/responses below api-version 2025-03-01-preview (probed live 2026-10-15),
|
||||
# so this call only succeeds if the proxy sends responses_api_version instead of api_version.
|
||||
model: Final = _register(
|
||||
proxy,
|
||||
resources,
|
||||
azure_openai_params(api_version="2024-10-01-preview", responses_api_version=AZURE_OPENAI_API_VERSION),
|
||||
prefix="e2e-responses-azure-override",
|
||||
)
|
||||
client: Final = sdk.openai(resources.key())
|
||||
|
||||
response: Final = client.responses.create(
|
||||
model=model,
|
||||
input="reply with one word",
|
||||
instructions=INSTRUCTIONS,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
assert response.output_text.strip(), (
|
||||
f"/responses over an azure deployment with responses_api_version set returned no output text: "
|
||||
f"{response.output!r}"
|
||||
)
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
|
|
|
|||
|
|
@ -1366,6 +1366,7 @@ class LiteLLMParamsBody(BaseModel):
|
|||
litellm_credential_name: str | None = None
|
||||
api_base: str | None = Field(default=None, repr=False)
|
||||
api_version: str | None = None
|
||||
responses_api_version: str | None = None
|
||||
realtime_protocol: str | None = None
|
||||
allowed_openai_params: list[str] | None = None
|
||||
drop_params: bool | None = None
|
||||
|
|
|
|||
|
|
@ -834,6 +834,61 @@ AZURE_AI_GPT_5_4_MINI_LATEST_STREAM: Final = CostTrackingTestCase(
|
|||
)
|
||||
|
||||
|
||||
# azure_ai/grok-4.6 (Grok on Azure AI Foundry reports reasoning_tokens outside completion_tokens, like xai)
|
||||
AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION: Final = CostTrackingTestCase(
|
||||
name="azure-ai-grok-4.6-reasoning_folded_into_completion",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="azure_ai/grok-4.6",
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"messages": [{"role": "user", "content": "azure-ai-grok-4.6-reasoning"}],
|
||||
"stream": False,
|
||||
},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "chatcmpl-$REQUEST_ID",
|
||||
"object": "chat.completion",
|
||||
"model": "grok-4.6",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {
|
||||
"prompt_tokens": 1840,
|
||||
"completion_tokens": 412,
|
||||
"total_tokens": 2552,
|
||||
"completion_tokens_details": {"reasoning_tokens": 300},
|
||||
},
|
||||
},
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.0108, input_cost=0.00368, output_cost=0.00712, prompt_tokens=1840, completion_tokens=712
|
||||
),
|
||||
)
|
||||
|
||||
AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION_STREAM: Final = CostTrackingTestCase(
|
||||
name="azure-ai-grok-4.6-reasoning_folded_into_completion-stream",
|
||||
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
|
||||
model="azure_ai/grok-4.6",
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"messages": [{"role": "user", "content": "azure-ai-grok-4.6-reasoning-stream"}],
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
response=SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=(
|
||||
'data: {"id":"chatcmpl-$REQUEST_ID","object":"chat.completion.chunk","created":1,"model":"grok-4.6","choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":null}]}\n\n',
|
||||
'data: {"id":"chatcmpl-$REQUEST_ID","object":"chat.completion.chunk","created":1,"model":"grok-4.6","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n',
|
||||
'data: {"id":"chatcmpl-$REQUEST_ID","object":"chat.completion.chunk","created":1,"model":"grok-4.6","choices":[],"usage":{"prompt_tokens":1840,"completion_tokens":412,"total_tokens":2552,"completion_tokens_details":{"reasoning_tokens":300}}}\n\n',
|
||||
"data: [DONE]\n\n",
|
||||
),
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.0108, input_cost=0.00368, output_cost=0.00712, prompt_tokens=1840, completion_tokens=712
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# azure_ai/claude-haiku-4-5 (Azure AI Foundry Anthropic, web search billed per server_tool_use.web_search_requests)
|
||||
AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH: Final = CostTrackingTestCase(
|
||||
name="azure-ai-claude-haiku-4-5-messages_web_search_two_requests",
|
||||
|
|
@ -1978,6 +2033,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
BEDROCK_INVOKE_HAIKU_STREAM,
|
||||
AZURE_AI_GPT_5_4_MINI_LATEST,
|
||||
AZURE_AI_GPT_5_4_MINI_LATEST_STREAM,
|
||||
AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION,
|
||||
AZURE_AI_GROK_4_6_REASONING_FOLDED_INTO_COMPLETION_STREAM,
|
||||
AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH,
|
||||
AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH_STREAM,
|
||||
AZURE_AI_CLAUDE_HAIKU_4_5_MESSAGES_WEB_SEARCH_ONE_REQUEST,
|
||||
|
|
|
|||
|
|
@ -764,6 +764,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"input_cost_per_token": 7.5e-07,
|
||||
"output_cost_per_token": 4.5e-06,
|
||||
},
|
||||
"azure_ai/grok-4.6": {
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 1e-05,
|
||||
},
|
||||
"azure_ai/claude-haiku-4-5": {
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
from typing import Final
|
||||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.openai_wire import chat_reply
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_DEPLOYMENT: Final = "gpt-6.1-sol"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "no reasoning please"}]
|
||||
|
||||
|
||||
def _reply(_request: Request) -> Reply:
|
||||
return chat_reply("chatcmpl-effort-none", _DEPLOYMENT, "answered without reasoning", stream=False)
|
||||
|
||||
|
||||
def test_reasoning_effort_none_is_rejected_by_litellm_before_reaching_azure(gateway: Gateway) -> None:
|
||||
with wire_server(_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"azure/{_DEPLOYMENT}",
|
||||
api_base=wire.url,
|
||||
api_key="synthetic-azure-key",
|
||||
api_version="2025-04-01-preview",
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": _MESSAGES, "reasoning_effort": "none"},
|
||||
key=scenario.key(),
|
||||
)
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert "reasoning_effort='none'" in response.text, response.text
|
||||
assert wire.drain() == ()
|
||||
|
||||
|
||||
def test_reasoning_effort_none_is_dropped_when_the_deployment_drops_params(gateway: Gateway) -> None:
|
||||
with wire_server(_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"azure/{_DEPLOYMENT}",
|
||||
api_base=wire.url,
|
||||
api_key="synthetic-azure-key",
|
||||
api_version="2025-04-01-preview",
|
||||
drop_params=True,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": _MESSAGES, "reasoning_effort": "none"},
|
||||
key=scenario.key(),
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert "answered without reasoning" in response.text, response.text
|
||||
received: Final = wire.drain()
|
||||
assert [request.method for request in received] == ["POST"], [request.target for request in received]
|
||||
body: Final = _JSON_OBJECT.validate_json(received[0].body)
|
||||
assert "reasoning_effort" not in body and "reasoning" not in body, body
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
from typing import Final
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from integration._support.client import Gateway, Scenario
|
||||
from integration._support.openai_wire import chat_reply, responses_reply
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_DEPLOYMENT: Final = "gpt-5.4-nano"
|
||||
_DATED_GA_API_VERSION: Final = "2024-10-21"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _reply(request: Request) -> Reply:
|
||||
if urlsplit(request.target).path.endswith("/responses"):
|
||||
return responses_reply("resp_azure_version", _DEPLOYMENT, "azure answered", stream=False)
|
||||
return chat_reply("chatcmpl-azure-version", _DEPLOYMENT, "azure answered", stream=False)
|
||||
|
||||
|
||||
def _dated_azure_model(scenario: Scenario, wire: Wire, model: str, responses_api_version: str | None = None) -> str:
|
||||
if responses_api_version is None:
|
||||
return scenario.model(
|
||||
model=model, api_base=wire.url, api_key="synthetic-azure-key", api_version=_DATED_GA_API_VERSION
|
||||
)
|
||||
return scenario.model(
|
||||
model=model,
|
||||
api_base=wire.url,
|
||||
api_key="synthetic-azure-key",
|
||||
api_version=_DATED_GA_API_VERSION,
|
||||
responses_api_version=responses_api_version,
|
||||
)
|
||||
|
||||
|
||||
def _only_post(wire: Wire) -> tuple[str, dict[str, str], dict[str, JsonValue]]:
|
||||
posts: Final = tuple(request for request in wire.drain() if request.method == "POST")
|
||||
assert len(posts) == 1, [request.target for request in posts]
|
||||
target: Final = urlsplit(posts[0].target)
|
||||
query: Final = {name: values[-1] for name, values in parse_qs(target.query).items()}
|
||||
return target.path, query, _JSON_OBJECT.validate_json(posts[0].body)
|
||||
|
||||
|
||||
def test_chat_keeps_the_dated_api_version_and_never_sends_responses_api_version(gateway: Gateway) -> None:
|
||||
with wire_server(_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _dated_azure_model(scenario, wire, f"azure/{_DEPLOYMENT}", responses_api_version="preview")
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": "dated chat"}]},
|
||||
key=scenario.key(),
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
path, query, body = _only_post(wire)
|
||||
assert (path, query) == (
|
||||
f"/openai/deployments/{_DEPLOYMENT}/chat/completions",
|
||||
{"api-version": _DATED_GA_API_VERSION},
|
||||
)
|
||||
assert body == {
|
||||
"model": _DEPLOYMENT,
|
||||
"messages": [{"role": "user", "content": "dated chat"}],
|
||||
"stream": False,
|
||||
}, body
|
||||
|
||||
|
||||
def test_responses_on_a_dated_ga_deployment_reaches_the_v1_responses_route(gateway: Gateway) -> None:
|
||||
with wire_server(_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _dated_azure_model(scenario, wire, f"azure/{_DEPLOYMENT}")
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": "dated responses"}, key=scenario.key()
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
path, query, body = _only_post(wire)
|
||||
assert (path, query) == ("/openai/v1/responses", {"api-version": "preview"})
|
||||
assert body["input"] == "dated responses", body
|
||||
|
||||
|
||||
def test_responses_api_version_overrides_the_chat_api_version_and_stays_out_of_the_body(gateway: Gateway) -> None:
|
||||
with wire_server(_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _dated_azure_model(
|
||||
scenario, wire, f"azure/{_DEPLOYMENT}", responses_api_version="2025-04-01-preview"
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": "override responses"}, key=scenario.key()
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
path, query, body = _only_post(wire)
|
||||
assert (path, query) == ("/openai/responses", {"api-version": "2025-04-01-preview"})
|
||||
assert "responses_api_version" not in body and "api_version" not in body, body
|
||||
|
||||
|
||||
def test_chat_bridged_through_azure_responses_on_a_dated_ga_deployment_reaches_v1_responses(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with wire_server(_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _dated_azure_model(scenario, wire, f"azure/responses/{_DEPLOYMENT}")
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": "bridged chat"}]},
|
||||
key=scenario.key(),
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert "azure answered" in response.text, response.text
|
||||
path, query, _ = _only_post(wire)
|
||||
assert (path, query) == ("/openai/v1/responses", {"api-version": "preview"})
|
||||
71
tests/integration/routing/test_wildcard_model_listing.py
Normal file
71
tests/integration/routing/test_wildcard_model_listing.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import uuid
|
||||
from collections import Counter
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, gateway_from_environment
|
||||
from integration._support.process import owned_proxy
|
||||
from pydantic import JsonValue
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(180)
|
||||
|
||||
_PRICED_DEPLOYMENT_ID: Final = f"priced-deployment-{uuid.uuid4().hex}"
|
||||
_REGISTERED_AI21_CHAT_MODEL: Final = f"jamba-house-{uuid.uuid4().hex}"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def candidate(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("wildcard-listing")
|
||||
with gateway_from_environment() as base:
|
||||
credentials: Final = {"api_base": f"{base.upstream_url}/v1", "api_key": "synthetic-listing-key"}
|
||||
config: Final = {
|
||||
**yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()),
|
||||
"model_list": [
|
||||
{"model_name": "openai/*", "litellm_params": {"model": "openai/*", **credentials}},
|
||||
{
|
||||
"model_name": "priced-chat",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"input_cost_per_token": 0.000001,
|
||||
"output_cost_per_token": 0.000002,
|
||||
**credentials,
|
||||
},
|
||||
"model_info": {"id": _PRICED_DEPLOYMENT_ID},
|
||||
},
|
||||
{"model_name": "ai21/*", "litellm_params": {"model": "ai21/*", **credentials}},
|
||||
{
|
||||
"model_name": "house-jamba",
|
||||
"litellm_params": {"model": f"ai21/{_REGISTERED_AI21_CHAT_MODEL}", **credentials},
|
||||
"model_info": {"litellm_provider": "ai21", "mode": "chat"},
|
||||
},
|
||||
],
|
||||
}
|
||||
path: Final = directory / "wildcard-listing.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with owned_proxy(base, directory, {}, config=path) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
def _listed_ids(gateway: Gateway) -> list[JsonValue]:
|
||||
data: Final = gateway.get("/v1/models")["data"]
|
||||
assert isinstance(data, list), data
|
||||
return [entry["id"] for entry in data if isinstance(entry, dict)]
|
||||
|
||||
|
||||
def test_wildcard_listing_after_a_priced_call_names_each_model_once_and_no_deployment_id(candidate: Gateway) -> None:
|
||||
candidate.chat("priced-chat")
|
||||
|
||||
listed: Final = _listed_ids(candidate)
|
||||
|
||||
assert "openai/gpt-4o-mini" in listed and "priced-chat" in listed, listed
|
||||
assert {model: count for model, count in Counter(listed).items() if count > 1} == {}, listed
|
||||
assert [model for model in listed if _PRICED_DEPLOYMENT_ID in str(model)] == [], listed
|
||||
|
||||
|
||||
def test_ai21_wildcard_listing_names_a_chat_model_registered_from_the_config(candidate: Gateway) -> None:
|
||||
listed: Final = _listed_ids(candidate)
|
||||
|
||||
assert f"ai21/{_REGISTERED_AI21_CHAT_MODEL}" in listed, listed
|
||||
|
|
@ -753,3 +753,33 @@ def test_azure_deployment_id_selects_chat_completions_endpoint(respx_mock: MockR
|
|||
assert route.call_count == 1
|
||||
assert route.calls[0].request.url.path == "/openai/deployments/deployment-gpt-4o/chat/completions"
|
||||
assert response.choices[0].message.content == "Hello"
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_responses_api_version_leaves_the_chat_route_and_body_untouched(respx_mock: MockRouter) -> None:
|
||||
url: Final = "https://example-resource.openai.azure.com/openai/deployments/gpt-5.4-nano/chat/completions"
|
||||
payload: Final = {
|
||||
"id": "chatcmpl-dated",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": "gpt-5.4-nano",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
|
||||
}
|
||||
route: Final = respx_mock.post(url__startswith=url).mock(return_value=httpx.Response(200, json=payload))
|
||||
messages: Final = [{"role": "user", "content": "Hello"}]
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="azure/gpt-5.4-nano",
|
||||
messages=messages,
|
||||
api_base="https://example-resource.openai.azure.com",
|
||||
api_key="azure-test-key",
|
||||
api_version="2024-12-01-preview",
|
||||
responses_api_version="preview",
|
||||
)
|
||||
request: Final = route.calls[0].request
|
||||
|
||||
assert route.call_count == 1
|
||||
assert request.url.params["api-version"] == "2024-12-01-preview"
|
||||
assert json.loads(request.content) == {"model": "gpt-5.4-nano", "messages": messages}
|
||||
assert response.choices[0].message.content == "Hello"
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
import json
|
||||
from collections.abc import Callable
|
||||
from copy import deepcopy
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
|
||||
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.azure.responses.o_series_transformation import (
|
||||
|
|
@ -282,16 +286,25 @@ class TestAzureResponsesAPIConfig:
|
|||
== "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest"
|
||||
)
|
||||
|
||||
# Test with date-based version - should use openai/responses
|
||||
result_date = self.config.get_complete_url(
|
||||
result_dated_preview = self.config.get_complete_url(
|
||||
api_base=base_url,
|
||||
litellm_params={"api_version": "2025-01-01"},
|
||||
litellm_params={"api_version": "2025-04-01-preview"},
|
||||
)
|
||||
assert (
|
||||
result_date
|
||||
== "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01"
|
||||
result_dated_preview
|
||||
== "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("api_version", ["2024-10-21", "2024-06-01", "2025-01-01"])
|
||||
def test_dated_ga_api_version_routes_responses_like_an_unset_one(self, api_version: str):
|
||||
base_url: Final = "https://litellm8397336933.openai.azure.com"
|
||||
|
||||
dated: Final = self.config.get_complete_url(api_base=base_url, litellm_params={"api_version": api_version})
|
||||
unset: Final = self.config.get_complete_url(api_base=base_url, litellm_params={})
|
||||
|
||||
assert dated == unset
|
||||
assert httpx.URL(dated).params["api-version"] == AZURE_DEFAULT_RESPONSES_API_VERSION
|
||||
|
||||
def test_azure_get_complete_url_with_default_api_version(self):
|
||||
"""Test Azure get_complete_url uses default API version when none is provided"""
|
||||
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
|
||||
|
|
@ -320,6 +333,64 @@ class TestAzureResponsesAPIConfig:
|
|||
)
|
||||
assert result_none_version == expected_url
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("api_version", "responses_api_version", "expected_path"),
|
||||
[
|
||||
("2024-10-21", "2025-04-01-preview", "/openai/responses?api-version=2025-04-01-preview"),
|
||||
("2025-04-01-preview", "v1", "/openai/v1/responses?api-version=v1"),
|
||||
(None, "2025-04-01-preview", "/openai/responses?api-version=2025-04-01-preview"),
|
||||
],
|
||||
)
|
||||
def test_responses_api_version_wins_over_api_version(
|
||||
self, api_version: str | None, responses_api_version: str, expected_path: str
|
||||
):
|
||||
base_url: Final = "https://litellm8397336933.openai.azure.com"
|
||||
|
||||
url: Final = self.config.get_complete_url(
|
||||
api_base=base_url,
|
||||
litellm_params={"api_version": api_version, "responses_api_version": responses_api_version},
|
||||
)
|
||||
|
||||
assert url == f"{base_url}{expected_path}"
|
||||
|
||||
@pytest.mark.parametrize("responses_api_version", [None, "v1"])
|
||||
def test_api_version_in_api_base_query_wins_over_a_dated_ga_api_version(self, responses_api_version: str | None):
|
||||
api_base: Final = "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview"
|
||||
|
||||
url: Final = self.config.get_complete_url(
|
||||
api_base=api_base,
|
||||
litellm_params={"api_version": "2024-10-21", "responses_api_version": responses_api_version},
|
||||
)
|
||||
|
||||
assert url == api_base
|
||||
|
||||
def test_api_version_in_env_api_base_query_wins_over_a_dated_ga_api_version(self, monkeypatch: pytest.MonkeyPatch):
|
||||
env_api_base: Final = (
|
||||
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview"
|
||||
)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.setenv("AZURE_API_BASE", env_api_base)
|
||||
|
||||
url: Final = self.config.get_complete_url(api_base=None, litellm_params={"api_version": "2024-10-21"})
|
||||
|
||||
assert url == env_api_base
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("api_base", "expected_path"),
|
||||
[
|
||||
("https://apim.example.com/openai-gw/team", "/openai-gw/team/openai/v1/responses"),
|
||||
("https://apim.example.com/openai/team-a", "/openai/team-a/openai/v1/responses"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("litellm_params", [{"api_version": "2024-10-21"}, {"responses_api_version": "v1"}])
|
||||
def test_v1_route_only_rewrites_the_responses_route_segment_of_a_gateway_api_base(
|
||||
self, api_base: str, expected_path: str, litellm_params: dict[str, str]
|
||||
):
|
||||
url: Final = httpx.URL(self.config.get_complete_url(api_base=api_base, litellm_params=litellm_params))
|
||||
|
||||
assert url.host == "apim.example.com"
|
||||
assert url.path == expected_path
|
||||
|
||||
def test_azure_cancel_response_api_request(self):
|
||||
"""Test Azure cancel response API request transformation"""
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
|
@ -684,6 +755,207 @@ def test_azure_responses_gpt6_astra_rejects_temperature_while_reasoning(local_mo
|
|||
)
|
||||
|
||||
|
||||
_COMPLETED_RESPONSE: Final = {
|
||||
"id": "resp_dated_chat_deployment",
|
||||
"object": "response",
|
||||
"created_at": 1734366691,
|
||||
"status": "completed",
|
||||
"model": "gpt-5.4-nano",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Done.", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
"error": None,
|
||||
"incomplete_details": None,
|
||||
"instructions": None,
|
||||
"metadata": None,
|
||||
"temperature": None,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"top_p": None,
|
||||
"max_output_tokens": None,
|
||||
"previous_response_id": None,
|
||||
"reasoning": None,
|
||||
"truncation": None,
|
||||
"user": None,
|
||||
}
|
||||
|
||||
|
||||
_AZURE_RESOURCE: Final = "https://example-resource.openai.azure.com"
|
||||
|
||||
|
||||
def _unset_api_version_responses_url() -> httpx.URL:
|
||||
return httpx.URL(AzureOpenAIResponsesAPIConfig().get_complete_url(api_base=_AZURE_RESOURCE, litellm_params={}))
|
||||
|
||||
|
||||
def _azure_responses_create(api_version: str) -> str | None:
|
||||
response: Final = litellm.responses(
|
||||
model="azure/gpt-5.4-nano",
|
||||
input="hi",
|
||||
api_base=_AZURE_RESOURCE,
|
||||
api_key="azure-test-key",
|
||||
api_version=api_version,
|
||||
)
|
||||
return response.output[0].content[0].text
|
||||
|
||||
|
||||
def _azure_chat_bridged_to_responses(api_version: str) -> str | None:
|
||||
response: Final = litellm.completion(
|
||||
model="azure/responses/gpt-5.4-nano",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base=_AZURE_RESOURCE,
|
||||
api_key="azure-test-key",
|
||||
api_version=api_version,
|
||||
)
|
||||
return response.choices[0].message.content
|
||||
|
||||
|
||||
@pytest.mark.parametrize("call_responses", [_azure_responses_create, _azure_chat_bridged_to_responses])
|
||||
def test_dated_ga_api_version_sends_responses_to_the_unset_api_version_url(
|
||||
respx_mock: respx.MockRouter, call_responses: Callable[[str], str | None]
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(host="example-resource.openai.azure.com").mock(
|
||||
return_value=httpx.Response(200, json=_COMPLETED_RESPONSE)
|
||||
)
|
||||
|
||||
text: Final = call_responses("2024-10-21")
|
||||
|
||||
assert text == "Done."
|
||||
assert route.call_count == 1
|
||||
assert route.calls[0].request.url == _unset_api_version_responses_url()
|
||||
|
||||
|
||||
_LIFECYCLE_BODIES: Final = {
|
||||
"get": _COMPLETED_RESPONSE,
|
||||
"delete": {"id": "resp_dated", "object": "response", "deleted": True},
|
||||
"cancel": {**_COMPLETED_RESPONSE, "status": "cancelled"},
|
||||
"input_items": {"object": "list", "data": [], "first_id": None, "last_id": None, "has_more": False},
|
||||
}
|
||||
|
||||
|
||||
def _call_lifecycle(operation: str) -> object:
|
||||
kwargs: Final = {
|
||||
"response_id": "resp_dated",
|
||||
"custom_llm_provider": "azure",
|
||||
"api_base": _AZURE_RESOURCE,
|
||||
"api_key": "azure-test-key",
|
||||
"api_version": "2024-10-21",
|
||||
}
|
||||
match operation:
|
||||
case "get":
|
||||
return litellm.get_responses(**kwargs)
|
||||
case "delete":
|
||||
return litellm.delete_responses(**kwargs)
|
||||
case "cancel":
|
||||
return litellm.cancel_responses(**kwargs)
|
||||
case _:
|
||||
return litellm.list_input_items(**kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("operation", "path_suffix"),
|
||||
[("get", ""), ("delete", ""), ("cancel", "/cancel"), ("input_items", "/input_items")],
|
||||
)
|
||||
def test_dated_ga_api_version_sends_response_lifecycle_calls_to_the_unset_api_version_route(
|
||||
respx_mock: respx.MockRouter, operation: str, path_suffix: str
|
||||
) -> None:
|
||||
route: Final = respx_mock.route(host="example-resource.openai.azure.com").mock(
|
||||
return_value=httpx.Response(200, json=_LIFECYCLE_BODIES[operation])
|
||||
)
|
||||
|
||||
_call_lifecycle(operation)
|
||||
|
||||
unset_url: Final = _unset_api_version_responses_url()
|
||||
sent_url: Final = route.calls[0].request.url
|
||||
assert route.call_count == 1
|
||||
assert sent_url.path == f"{unset_url.path}/resp_dated{path_suffix}"
|
||||
assert sent_url.params["api-version"] == unset_url.params["api-version"]
|
||||
|
||||
|
||||
def test_chat_completion_keeps_the_dated_ga_api_version(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(
|
||||
f"{_AZURE_RESOURCE}/openai/deployments/gpt-5.4-nano/chat/completions", params={"api-version": "2024-10-21"}
|
||||
).mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-dated",
|
||||
"object": "chat.completion",
|
||||
"created": 1734366691,
|
||||
"model": "gpt-5.4-nano",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Done."}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="azure/gpt-5.4-nano",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base=_AZURE_RESOURCE,
|
||||
api_key="azure-test-key",
|
||||
api_version="2024-10-21",
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "Done."
|
||||
assert route.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_responses_api_version_routes_a_dated_chat_deployment_to_its_own_responses_version(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(
|
||||
"https://example-resource.openai.azure.com/openai/responses", params={"api-version": "2025-04-01-preview"}
|
||||
).mock(return_value=httpx.Response(200, json=_COMPLETED_RESPONSE))
|
||||
|
||||
response: Final = litellm.responses(
|
||||
model="azure/gpt-5.4-nano",
|
||||
input="hi",
|
||||
api_base="https://example-resource.openai.azure.com",
|
||||
api_key="azure-test-key",
|
||||
api_version="2024-10-21",
|
||||
responses_api_version="2025-04-01-preview",
|
||||
)
|
||||
request_body: Final = json.loads(route.calls[0].request.content)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert response.output[0].content[0].text == "Done."
|
||||
assert request_body["model"] == "gpt-5.4-nano"
|
||||
assert "responses_api_version" not in request_body
|
||||
assert "api_version" not in request_body
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_responses_api_version_reaches_the_url_when_chat_completion_bridges_to_responses(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(
|
||||
"https://example-resource.openai.azure.com/openai/v1/responses", params={"api-version": "v1"}
|
||||
).mock(return_value=httpx.Response(200, json=_COMPLETED_RESPONSE))
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="azure/responses/gpt-5.4-nano",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base="https://example-resource.openai.azure.com",
|
||||
api_key="azure-test-key",
|
||||
api_version="2024-10-21",
|
||||
responses_api_version="v1",
|
||||
)
|
||||
request_body: Final = json.loads(route.calls[0].request.content)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert response.choices[0].message.content == "Done."
|
||||
assert "responses_api_version" not in request_body
|
||||
|
||||
|
||||
def test_azure_responses_sends_the_deployment_name_when_azure_ai_prefix_survives_provider_remap():
|
||||
request = AzureOpenAIResponsesAPIConfig().transform_responses_api_request(
|
||||
model="azure_ai/gpt-5.4-nano",
|
||||
|
|
|
|||
|
|
@ -345,6 +345,80 @@ def test_azure_ai_grok_stop_parameter_handling():
|
|||
assert "stop" in gpt_params, "GPT models should support stop parameter"
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_azure_ai_grok_bills_reasoning_tokens_reported_outside_completion_tokens(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
api_url: Final = "https://example-resource.services.ai.azure.com/models/chat/completions"
|
||||
route: Final = respx_mock.post(url__startswith=api_url).mock(
|
||||
return_value=Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-grok",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": "grok-4.7",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "42"}, "finish_reason": "stop"}],
|
||||
"usage": {
|
||||
"prompt_tokens": 14,
|
||||
"completion_tokens": 10,
|
||||
"total_tokens": 336,
|
||||
"completion_tokens_details": {"reasoning_tokens": 312},
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="azure_ai/grok-4.7",
|
||||
messages=[{"role": "user", "content": "What is the meaning of life?"}],
|
||||
api_base="https://example-resource.services.ai.azure.com/models",
|
||||
api_key="azure-ai-test-key",
|
||||
)
|
||||
model_info: Final = litellm.get_model_info(model="grok-4.7", custom_llm_provider="azure_ai")
|
||||
expected_cost: Final = 14 * model_info["input_cost_per_token"] + 322 * model_info["output_cost_per_token"]
|
||||
|
||||
assert route.call_count == 1
|
||||
assert response.usage.completion_tokens == 322
|
||||
assert response.usage.total_tokens == response.usage.prompt_tokens + response.usage.completion_tokens
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(expected_cost, rel=1e-9)
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_azure_ai_grok_stream_folds_reasoning_tokens_into_the_usage_chunk(respx_mock: respx.MockRouter) -> None:
|
||||
api_url: Final = "https://example-resource.services.ai.azure.com/models/chat/completions"
|
||||
stream_body: Final = (
|
||||
'data: {"id":"chatcmpl-grok","object":"chat.completion.chunk","created":1700000000,"model":"grok-4.7",'
|
||||
'"choices":[{"index":0,"delta":{"role":"assistant","content":"42"},"finish_reason":null}]}\n\n'
|
||||
'data: {"id":"chatcmpl-grok","object":"chat.completion.chunk","created":1700000000,"model":"grok-4.7",'
|
||||
'"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n'
|
||||
'data: {"id":"chatcmpl-grok","object":"chat.completion.chunk","created":1700000000,"model":"grok-4.7",'
|
||||
'"choices":[],"usage":{"prompt_tokens":14,"completion_tokens":10,"total_tokens":336,'
|
||||
'"completion_tokens_details":{"reasoning_tokens":312}}}\n\n'
|
||||
"data: [DONE]\n\n"
|
||||
)
|
||||
route: Final = respx_mock.post(url__startswith=api_url).mock(
|
||||
return_value=Response(200, text=stream_body, headers={"content-type": "text/event-stream"})
|
||||
)
|
||||
|
||||
chunks: Final = tuple(
|
||||
litellm.completion(
|
||||
model="azure_ai/grok-4.7",
|
||||
messages=[{"role": "user", "content": "What is the meaning of life?"}],
|
||||
api_base="https://example-resource.services.ai.azure.com/models",
|
||||
api_key="azure-ai-test-key",
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
)
|
||||
usage_chunks: Final = [chunk.usage for chunk in chunks if getattr(chunk, "usage", None) is not None]
|
||||
|
||||
assert route.call_count == 1
|
||||
assert len(usage_chunks) == 1
|
||||
assert usage_chunks[0].completion_tokens == 322
|
||||
assert usage_chunks[0].total_tokens == usage_chunks[0].prompt_tokens + usage_chunks[0].completion_tokens
|
||||
|
||||
|
||||
def test_azure_model_router_response_shows_actual_model():
|
||||
"""
|
||||
Test that Azure Model Router returns the actual model used in the response,
|
||||
|
|
|
|||
|
|
@ -1434,11 +1434,11 @@ class TestAzureResponsesAPIConfig:
|
|||
# Test with date-based version - should use openai/responses
|
||||
result_date = self.config.get_complete_url(
|
||||
api_base=base_url,
|
||||
litellm_params={"api_version": "2025-01-01"},
|
||||
litellm_params={"api_version": "2025-04-01-preview"},
|
||||
)
|
||||
assert (
|
||||
result_date
|
||||
== "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01"
|
||||
== "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview"
|
||||
)
|
||||
|
||||
def test_azure_transform_then_normalize_strips_custom_tool_call_namespace(self):
|
||||
|
|
|
|||
|
|
@ -932,6 +932,26 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion():
|
|||
assert fake_model not in litellm.models_by_provider["vertex_ai"]
|
||||
|
||||
|
||||
def test_wildcard_lists_a_model_once_when_registered_both_bare_and_provider_prefixed():
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
|
||||
bare: Final = "gpt-dedupe-regression"
|
||||
entry: Final = {"litellm_provider": "openai", "mode": "chat"}
|
||||
try:
|
||||
litellm.register_model({bare: entry, f"openai/{bare}": entry}, persist_across_reloads=False)
|
||||
|
||||
listed: Final = get_known_models_from_wildcard("openai/*")
|
||||
|
||||
assert listed.count(f"openai/{bare}") == 1
|
||||
finally:
|
||||
litellm.model_cost.pop(bare, None)
|
||||
litellm.model_cost.pop(f"openai/{bare}", None)
|
||||
litellm.open_ai_chat_completion_models.discard(bare)
|
||||
litellm.open_ai_chat_completion_models.discard(f"openai/{bare}")
|
||||
litellm.add_known_models(model_cost_map={})
|
||||
|
||||
|
||||
def test_get_complete_model_list_drops_no_default_models_sentinel():
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
|
|
|
|||
|
|
@ -5357,6 +5357,7 @@ class TestRouterPreRoutingSharedAliasName:
|
|||
"api_key": "sk-marker",
|
||||
"api_base": "https://marker.example/v1",
|
||||
"api_version": "2024-01-01",
|
||||
"responses_api_version": "v1",
|
||||
},
|
||||
}
|
||||
router = Router(model_list=[marker_with_connection_params, self._tier_entry()])
|
||||
|
|
@ -5372,6 +5373,7 @@ class TestRouterPreRoutingSharedAliasName:
|
|||
assert "api_base" not in request_kwargs
|
||||
assert "api_key" not in request_kwargs
|
||||
assert "api_version" not in request_kwargs
|
||||
assert "responses_api_version" not in request_kwargs
|
||||
assert request_kwargs["drop_params"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -419,6 +419,24 @@ def test_azure_gpt_6_1_sol_data_zone_rows_price_requests_at_the_regional_premium
|
|||
assert total_cost(f"azure/{region}/gpt-6.1-sol") == pytest.approx(global_cost * premium)
|
||||
|
||||
|
||||
AZURE_GPT_6_1_SOL_ROWS: Final = (
|
||||
"azure/gpt-6.1-sol",
|
||||
"azure/gpt-6.1-sol-2026-09-29",
|
||||
"azure/us/gpt-6.1-sol",
|
||||
"azure/eu/gpt-6.1-sol",
|
||||
"azure/apac/gpt-6.1-sol",
|
||||
"azure_ai/gpt-6.1-sol",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("row", AZURE_GPT_6_1_SOL_ROWS)
|
||||
def test_azure_gpt_6_1_sol_rows_accept_the_same_reasoning_efforts_as_the_openai_row(prices: dict, row: str):
|
||||
openai_efforts: Final = resolve_supported_reasoning_efforts(prices["gpt-6.1-sol"], deployment_is_mapped=True)
|
||||
assert openai_efforts
|
||||
|
||||
assert resolve_supported_reasoning_efforts(prices[row], deployment_is_mapped=True) == openai_efforts
|
||||
|
||||
|
||||
# https://ai.google.dev/gemini-api/docs/models/deep-research-preview-04-2026
|
||||
@pytest.mark.parametrize("model", ["gemini/deep-research-preview-04-2026", "gemini/deep-research-max-preview-04-2026"])
|
||||
def test_gemini_deep_research_previews_accept_the_documented_input_window(prices: dict, model: str):
|
||||
|
|
|
|||
|
|
@ -2418,6 +2418,87 @@ def test_register_model_openrouter_without_slash():
|
|||
assert "openai/gpt-4-turbo" in litellm.openrouter_models
|
||||
|
||||
|
||||
def test_register_model_makes_the_model_resolvable_and_wildcard_visible_without_a_restart():
|
||||
model_name: Final = "my-tuned-gemini-regression"
|
||||
try:
|
||||
litellm.register_model(
|
||||
{
|
||||
model_name: {
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0.000001,
|
||||
"output_cost_per_token": 0.000002,
|
||||
}
|
||||
},
|
||||
persist_across_reloads=False,
|
||||
)
|
||||
|
||||
_, provider, _, _ = litellm.get_llm_provider(model=model_name)
|
||||
|
||||
assert provider == "vertex_ai"
|
||||
assert model_name in litellm.models_by_provider["vertex_ai"]
|
||||
finally:
|
||||
litellm.model_cost.pop(model_name, None)
|
||||
litellm.vertex_language_models.discard(model_name)
|
||||
litellm.models_by_provider["vertex_ai"].discard(model_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_provider", "wildcard", "provider_set"),
|
||||
[
|
||||
("ai21", "ai21/*", "ai21_chat_models"),
|
||||
("vertex_ai-code-chat-models", "vertex_ai/*", "vertex_code_chat_models"),
|
||||
],
|
||||
)
|
||||
def test_registered_chat_model_lists_under_its_provider_wildcard(
|
||||
litellm_provider: str, wildcard: str, provider_set: str
|
||||
) -> None:
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
|
||||
model_name: Final = f"chat-listing-regression-{litellm_provider}"
|
||||
try:
|
||||
litellm.register_model(
|
||||
{model_name: {"litellm_provider": litellm_provider, "mode": "chat"}}, persist_across_reloads=False
|
||||
)
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(wildcard)
|
||||
|
||||
assert f"{wildcard.removesuffix('*')}{model_name}" in listed
|
||||
finally:
|
||||
litellm.model_cost.pop(model_name, None)
|
||||
getattr(litellm, provider_set).discard(model_name)
|
||||
litellm.add_known_models(model_cost_map={})
|
||||
|
||||
|
||||
def test_priced_router_deployment_id_stays_out_of_wildcard_model_listing():
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
|
||||
deployment_id: Final = "priced-deployment-id-regression"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "priced-gemini",
|
||||
"litellm_params": {
|
||||
"model": "gemini/gemini-2.5-flash",
|
||||
"api_key": "fake-key",
|
||||
"input_cost_per_token": 0.000001,
|
||||
"output_cost_per_token": 0.000002,
|
||||
},
|
||||
"model_info": {"id": deployment_id, "litellm_provider": "gemini", "mode": "chat"},
|
||||
},
|
||||
{"model_name": "gemini/*", "litellm_params": {"model": "gemini/*", "api_key": "fake-key"}},
|
||||
]
|
||||
)
|
||||
|
||||
router.completion(model="priced-gemini", messages=[{"role": "user", "content": "hi"}], mock_response="ok")
|
||||
listed: Final = get_known_models_from_wildcard("gemini/*")
|
||||
|
||||
assert "gemini/gemini-2.5-flash" in listed
|
||||
assert f"gemini/{deployment_id}" not in listed
|
||||
assert deployment_id not in litellm.models_by_provider["gemini"]
|
||||
assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.000001
|
||||
|
||||
|
||||
def test_reasoning_content_preserved_in_text_completion_wrapper():
|
||||
"""Ensure reasoning_content is copied from delta to text_choices."""
|
||||
chunk = ModelResponseStream(
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ CONNECTION_NAMES: Final = (
|
|||
"api_key",
|
||||
"api_base",
|
||||
"api_version",
|
||||
"responses_api_version",
|
||||
"region_name",
|
||||
"headers",
|
||||
"provider_specific_header",
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -35325,6 +35325,8 @@ export interface components {
|
|||
regional_processing_uplift_multiplier_eu?: number | null;
|
||||
/** Regional Processing Uplift Multiplier Us */
|
||||
regional_processing_uplift_multiplier_us?: number | null;
|
||||
/** Responses Api Version */
|
||||
responses_api_version?: string | null;
|
||||
/** Rpm */
|
||||
rpm?: number | null;
|
||||
/** S3 Access Key Id */
|
||||
|
|
@ -50550,6 +50552,8 @@ export interface components {
|
|||
regional_processing_uplift_multiplier_eu?: number | null;
|
||||
/** Regional Processing Uplift Multiplier Us */
|
||||
regional_processing_uplift_multiplier_us?: number | null;
|
||||
/** Responses Api Version */
|
||||
responses_api_version?: string | null;
|
||||
/** Rpm */
|
||||
rpm?: number | null;
|
||||
/** S3 Access Key Id */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue