scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 114

Responses API - add default api version for openai responses api calls + Openrouter - fix claude-sonnet-4 on openrouter + Azure - Handle openai/v1/responses

Browse source 
Responses API - add default api version for openai responses api calls + Openrouter - fix claude-sonnet-4 on openrouter + Azure - Handle `openai/v1/responses`
This commit is contained in:
Krish Dholakia 2025-08-18 18:59:28 -07:00 • committed by GitHub
commit 422447b7f1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 185 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/my-website/docs/proxy/config_settings.md
View file

@ -349,6 +349,7 @@ router_settings:
| AZURE_CODE_INTERPRETER_COST_PER_SESSION | Cost per session for Azure Code Interpreter service
| AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS | Input cost per 1K tokens for Azure Computer Use service
| AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS | Output cost per 1K tokens for Azure Computer Use service
| AZURE_DEFAULT_RESPONSES_API_VERSION | Version of the Azure Default Responses API being used. Default is "preview"
| AZURE_TENANT_ID | Tenant ID for Azure Active Directory
| AZURE_USERNAME | Username for Azure services, use in conjunction with AZURE_PASSWORD for azure ad token with basic username/password workflow
| AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow

3
litellm/constants.py
View file

@ -1,6 +1,9 @@
import os
from typing import List, Literal
AZURE_DEFAULT_RESPONSES_API_VERSION = str(
os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")
)
ROUTER_MAX_FALLBACKS = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
DEFAULT_BATCH_SIZE = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
DEFAULT_FLUSH_INTERVAL_SECONDS = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5))

69
litellm/llms/azure/common_utils.py
View file

@ -365,14 +365,16 @@ def get_azure_ad_token(
azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope)
except ValueError:
verbose_logger.debug("Azure AD Token Provider could not be used.")
#########################################################
# If litellm.enable_azure_ad_token_refresh is True and no other token provider is available,
# try to get DefaultAzureCredential provider
#########################################################
if azure_ad_token_provider is None and azure_ad_token is None:
azure_ad_token_provider = BaseAzureLLM._try_get_default_azure_credential_provider(
scope=scope,
azure_ad_token_provider = (
BaseAzureLLM._try_get_default_azure_credential_provider(
scope=scope,
)
)
# Execute the token provider to get the token if available
@ -403,27 +405,27 @@ class BaseAzureLLM(BaseOpenAILLM):
) -> Optional[Callable[[], str]]:
"""
Try to get DefaultAzureCredential provider
Args:
scope: Azure scope for the token
Returns:
Token provider callable if DefaultAzureCredential is enabled and available, None otherwise
"""
from litellm.types.secret_managers.get_azure_ad_token_provider import (
AzureCredentialType,
)
verbose_logger.debug(
"Attempting to use DefaultAzureCredential for Azure Auth"
)
verbose_logger.debug("Attempting to use DefaultAzureCredential for Azure Auth")
try:
azure_ad_token_provider = get_azure_ad_token_provider(
azure_scope=scope,
azure_credential=AzureCredentialType.DefaultAzureCredential,
)
verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential")
verbose_logger.debug(
"Successfully obtained Azure AD token provider using DefaultAzureCredential"
)
return azure_ad_token_provider
except Exception as e:
verbose_logger.debug(f"DefaultAzureCredential failed: {str(e)}")
@ -656,17 +658,17 @@ class BaseAzureLLM(BaseOpenAILLM):
else:
client = AzureOpenAI(**azure_client_params) # type: ignore
return client
@staticmethod
def _base_validate_azure_environment(
headers: dict, litellm_params: Optional[GenericLiteLLMParams]
headers: dict, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:
litellm_params = litellm_params or GenericLiteLLMParams()
# If api-key is already in headers, preserve it
if "api-key" in headers:
return headers
api_key = (
litellm_params.api_key
or litellm.api_key
@ -686,13 +688,24 @@ class BaseAzureLLM(BaseOpenAILLM):
headers["Authorization"] = f"Bearer {azure_ad_token}"
return headers
@staticmethod
def _get_base_azure_url(
api_base: Optional[str],
litellm_params: Optional[Union[GenericLiteLLMParams, Dict[str, Any]]],
route: Literal["/openai/responses", "/openai/vector_stores"]
route: Literal["/openai/responses", "/openai/vector_stores"],
default_api_version: Optional[Union[str, Literal["latest", "preview"]]] = None,
) -> str:
"""
Get the base Azure URL for the given route and API version.
Args:
api_base: The base URL of the Azure API.
litellm_params: The litellm parameters.
route: The route to the API.
default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route.
"""
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
if api_base is None:
raise ValueError(
@ -702,7 +715,10 @@ class BaseAzureLLM(BaseOpenAILLM):
# Extract api_version or use default
litellm_params = litellm_params or {}
api_version = cast(Optional[str], litellm_params.get("api_version"))
api_version = (
cast(Optional[str], litellm_params.get("api_version"))
or default_api_version
)
# Create a new dictionary with existing params
query_params = dict(original_url.params)
@ -710,27 +726,28 @@ class BaseAzureLLM(BaseOpenAILLM):
# Add api_version if needed
if "api-version" not in query_params and api_version:
query_params["api-version"] = api_version
# Add the path to the base URL
if route not in api_base:
new_url = _add_path_to_api_base(
api_base=api_base, ending_path=route
)
new_url = _add_path_to_api_base(api_base=api_base, ending_path=route)
else:
new_url = api_base
if BaseAzureLLM._is_azure_v1_api_version(api_version):
# ensure the request go to /openai/v1 and not just /openai
if "/openai/v1" not in new_url:
parsed_url = httpx.URL(new_url)
new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1")))
new_url = str(
parsed_url.copy_with(
path=parsed_url.path.replace("/openai", "/openai/v1")
)
)
# Use the new query_params dictionary
final_url = httpx.URL(new_url).copy_with(params=query_params)
return str(final_url)
@staticmethod
def _is_azure_v1_api_version(api_version: Optional[str]) -> bool:
if api_version is None:

7
litellm/llms/azure/responses/transformation.py
View file

@ -70,8 +70,13 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
- A complete URL string, e.g.,
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
"""
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
return BaseAzureLLM._get_base_azure_url(
api_base=api_base, litellm_params=litellm_params, route="/openai/responses"
api_base=api_base,
litellm_params=litellm_params,
route="/openai/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)
#########################################################

4
litellm/model_prices_and_context_window_backup.json
View file

@ -11476,9 +11476,9 @@
},
"openrouter/anthropic/claude-sonnet-4": {
"supports_computer_use": true,
"max_tokens": 8192,
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"input_cost_per_image": 0.0048,

9
litellm/proxy/_new_secret_config.yaml
View file

@ -4,6 +4,15 @@ model_list:
model: openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
- model_name: gpt-5-mini
litellm_params:
model: azure/gpt-5-mini
api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE")
api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY")
stream_timeout: 60
merge_reasoning_content_in_choices: true
model_info:
mode: chat
litellm_settings:
cache: true

4
model_prices_and_context_window.json
View file

@ -11476,9 +11476,9 @@
},
"openrouter/anthropic/claude-sonnet-4": {
"supports_computer_use": true,
"max_tokens": 8192,
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"input_cost_per_image": 0.0048,

14
tests/llm_translation/test_azure_openai.py
View file

@ -630,3 +630,17 @@ def test_azure_openai_responses_bridge():
== "test-azure-computer-use-preview"
)
assert mock_responses.call_args.kwargs["custom_llm_provider"] == "azure"
def test_azure_openai_gpt_5_responses_api():
from litellm import responses
litellm._turn_on_debug()
response = responses(
model="azure/gpt-5",
input="Hello world",
api_key=os.getenv("AZURE_SWEDEN_API_KEY"),
api_base=os.getenv("AZURE_SWEDEN_API_BASE"),
)
print(f"response: {response}")

140
tests/test_litellm/llms/azure/response/test_azure_transformation.py
View file

@ -8,10 +8,14 @@ sys.path.insert(
0, os.path.abspath("../../../../..")
) # Adds the parent directory to the system path
from unittest.mock import MagicMock
from litellm.llms.azure.responses.o_series_transformation import (
AzureOpenAIOSeriesResponsesAPIConfig,
)
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from litellm.llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig
from litellm.types.router import GenericLiteLLMParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
@pytest.mark.serial
@ -27,6 +31,7 @@ def test_validate_environment_api_key_within_litellm_params():
assert result == expected
@pytest.mark.serial
def test_validate_environment_api_key_within_litellm():
azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig()
@ -41,6 +46,7 @@ def test_validate_environment_api_key_within_litellm():
assert result == expected
@pytest.mark.serial
def test_validate_environment_azure_key_within_litellm():
azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig()
@ -55,6 +61,7 @@ def test_validate_environment_azure_key_within_litellm():
assert result == expected
@pytest.mark.serial
def test_validate_environment_azure_key_within_headers():
azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig()
@ -93,10 +100,10 @@ def test_azure_o_series_responses_api_supported_params():
"""Test that Azure OpenAI O-series responses API excludes temperature from supported parameters."""
config = AzureOpenAIOSeriesResponsesAPIConfig()
supported_params = config.get_supported_openai_params("o_series/gpt-o1")
# Temperature should not be in supported params for O-series models
assert "temperature" not in supported_params
# Other parameters should still be supported
assert "input" in supported_params
assert "max_output_tokens" in supported_params
@ -108,35 +115,32 @@ def test_azure_o_series_responses_api_supported_params():
def test_azure_o_series_responses_api_drop_temperature_param():
"""Test that temperature parameter is dropped when drop_params is True for O-series models."""
config = AzureOpenAIOSeriesResponsesAPIConfig()
# Create request params with temperature
request_params = ResponsesAPIOptionalRequestParams(
temperature=0.7,
max_output_tokens=1000,
stream=False,
top_p=0.9
temperature=0.7, max_output_tokens=1000, stream=False, top_p=0.9
)
# Test with drop_params=True
mapped_params_with_drop = config.map_openai_params(
response_api_optional_params=request_params,
model="o_series/gpt-o1",
drop_params=True
drop_params=True,
)
# Temperature should be dropped
assert "temperature" not in mapped_params_with_drop
# Other params should remain
assert mapped_params_with_drop["max_output_tokens"] == 1000
assert mapped_params_with_drop["top_p"] == 0.9
# Test with drop_params=False
mapped_params_without_drop = config.map_openai_params(
response_api_optional_params=request_params,
model="o_series/gpt-o1",
drop_params=False
drop_params=False,
)
# Temperature should still be present when drop_params=False
assert mapped_params_without_drop["temperature"] == 0.7
assert mapped_params_without_drop["max_output_tokens"] == 1000
@ -147,21 +151,19 @@ def test_azure_o_series_responses_api_drop_temperature_param():
def test_azure_o_series_responses_api_drop_params_no_temperature():
"""Test that map_openai_params works correctly when temperature is not present for O-series models."""
config = AzureOpenAIOSeriesResponsesAPIConfig()
# Create request params without temperature
request_params = ResponsesAPIOptionalRequestParams(
max_output_tokens=1000,
stream=False,
top_p=0.9
max_output_tokens=1000, stream=False, top_p=0.9
)
# Should work fine even with drop_params=True
mapped_params = config.map_openai_params(
response_api_optional_params=request_params,
model="o_series/gpt-o1",
drop_params=True
drop_params=True,
)
assert "temperature" not in mapped_params
assert mapped_params["max_output_tokens"] == 1000
assert mapped_params["top_p"] == 0.9
@ -172,10 +174,10 @@ def test_azure_regular_responses_api_supports_temperature():
"""Test that regular Azure OpenAI responses API (non-O-series) supports temperature parameter."""
config = AzureOpenAIResponsesAPIConfig()
supported_params = config.get_supported_openai_params("gpt-4o")
# Regular Azure models should support temperature
assert "temperature" in supported_params
# Other parameters should still be supported
assert "input" in supported_params
assert "max_output_tokens" in supported_params
@ -187,11 +189,11 @@ def test_azure_regular_responses_api_supports_temperature():
def test_o_series_model_detection():
"""Test that the O-series configuration correctly identifies O-series models."""
config = AzureOpenAIOSeriesResponsesAPIConfig()
# Test explicit o_series naming
assert config.is_o_series_model("o_series/gpt-o1") == True
assert config.is_o_series_model("azure/o_series/gpt-o3") == True
# Test regular models
assert config.is_o_series_model("gpt-4o") == False
assert config.is_o_series_model("gpt-3.5-turbo") == False
@ -200,28 +202,94 @@ def test_o_series_model_detection():
@pytest.mark.serial
def test_provider_config_manager_o_series_selection():
"""Test that ProviderConfigManager returns the correct config for O-series vs regular models."""
from litellm.utils import ProviderConfigManager
import litellm
from litellm.utils import ProviderConfigManager
# Test O-series model selection
o_series_config = ProviderConfigManager.get_provider_responses_api_config(
provider=litellm.LlmProviders.AZURE,
model="o_series/gpt-o1"
provider=litellm.LlmProviders.AZURE, model="o_series/gpt-o1"
)
assert isinstance(o_series_config, AzureOpenAIOSeriesResponsesAPIConfig)
# Test regular model selection
regular_config = ProviderConfigManager.get_provider_responses_api_config(
provider=litellm.LlmProviders.AZURE,
model="gpt-4o"
provider=litellm.LlmProviders.AZURE, model="gpt-4o"
)
assert isinstance(regular_config, AzureOpenAIResponsesAPIConfig)
assert not isinstance(regular_config, AzureOpenAIOSeriesResponsesAPIConfig)
# Test with no model specified (should default to regular)
default_config = ProviderConfigManager.get_provider_responses_api_config(
provider=litellm.LlmProviders.AZURE,
model=None
provider=litellm.LlmProviders.AZURE, model=None
)
assert isinstance(default_config, AzureOpenAIResponsesAPIConfig)
assert not isinstance(default_config, AzureOpenAIOSeriesResponsesAPIConfig)
class TestAzureResponsesAPIConfig:
def setup_method(self):
self.config = AzureOpenAIResponsesAPIConfig()
self.model = "gpt-4o"
self.logging_obj = MagicMock()
def test_azure_get_complete_url_with_version_types(self):
"""Test Azure get_complete_url with different API version types"""
base_url = "https://litellm8397336933.openai.azure.com"
# Test with preview version - should use openai/v1/responses
result_preview = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "preview"},
)
assert (
result_preview
== "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview"
)
# Test with latest version - should use openai/v1/responses
result_latest = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "latest"},
)
assert (
result_latest
== "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest"
)
# Test with date-based version - should use openai/responses
result_date = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "2025-01-01"},
)
assert (
result_date
== "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01"
)
def test_azure_get_complete_url_with_default_api_version(self):
"""Test Azure get_complete_url uses default API version when none is provided"""
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
base_url = "https://litellm8397336933.openai.azure.com"
# Test with no api_version provided - should use default
result_no_version = self.config.get_complete_url(
api_base=base_url,
litellm_params={},
)
expected_url = f"https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version={AZURE_DEFAULT_RESPONSES_API_VERSION}"
assert result_no_version == expected_url
# Test with empty litellm_params - should use default
result_empty_params = self.config.get_complete_url(
api_base=base_url,
litellm_params={},
)
assert result_empty_params == expected_url
# Test with None api_version - should use default
result_none_version = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": None},
)
assert result_none_version == expected_url

34
tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py
View file

@ -147,7 +147,7 @@ class TestOpenAIResponsesAPIConfig:
assert result.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
assert result.response.id == "resp_123"
@pytest.mark.serial
def test_validate_environment(self):
"""Test that validate_environment correctly sets the Authorization header"""
@ -283,38 +283,6 @@ class TestOpenAIResponsesAPIConfig:
assert result.type == "test"
class TestAzureResponsesAPIConfig:
def setup_method(self):
self.config = AzureOpenAIResponsesAPIConfig()
self.model = "gpt-4o"
self.logging_obj = MagicMock()
def test_azure_get_complete_url_with_version_types(self):
"""Test Azure get_complete_url with different API version types"""
base_url = "https://litellm8397336933.openai.azure.com"
# Test with preview version - should use openai/v1/responses
result_preview = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "preview"},
)
assert result_preview == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview"
# Test with latest version - should use openai/v1/responses
result_latest = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "latest"},
)
assert result_latest == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest"
# Test with date-based version - should use openai/responses
result_date = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "2025-01-01"},
)
assert result_date == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01"
class TestTransformListInputItemsRequest:
"""Test suite for transform_list_input_items_request function"""