scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

Merge pull request #19042 from BerriAI/litellm_staging_01_13_2026

Browse source 
Staging 01/13/2026
This commit is contained in:
Sameer Kankute 2026-01-14 21:29:09 +05:30 • committed by GitHub
commit cd2d381dd4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
35 changed files with 907 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/test-litellm.yml vendored
View file

@ -35,6 +35,7 @@ jobs:
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.18"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |
cd enterprise

3
Makefile
View file

@ -45,6 +45,7 @@ install-proxy-dev-ci:
install-test-deps: install-proxy-dev
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
poetry run pip install openapi-core
cd enterprise && poetry run pip install -e . && cd ..
install-helm-unittest:
@ -100,4 +101,4 @@ test-llm-translation-single: install-test-deps
@mkdir -p test-results
poetry run pytest tests/llm_translation/$(FILE) \
--junitxml=test-results/junit.xml \
-v --tb=short --maxfail=100 --timeout=300
-v --tb=short --maxfail=100 --timeout=300

1
enterprise/litellm_enterprise/proxy/__init__.py Normal file
View file

@ -0,0 +1 @@
# Package marker for enterprise proxy components.

1
enterprise/litellm_enterprise/proxy/common_utils/__init__.py Normal file
View file

@ -0,0 +1 @@
# Package marker for enterprise proxy common utilities.

21
litellm/integrations/langsmith.py
View file

@ -134,6 +134,13 @@ class LangsmithLogger(CustomBatchLogger):
"metadata"
] # ensure logged metadata is json serializable
extra_metadata = dict(metadata)
requester_metadata = extra_metadata.get("requester_metadata")
if requester_metadata and isinstance(requester_metadata, dict):
for key in ("session_id", "thread_id", "conversation_id"):
if key in requester_metadata and key not in extra_metadata:
extra_metadata[key] = requester_metadata[key]
data = {
"name": run_name,
"run_type": "llm", # this should always be llm, since litellm always logs llm calls. Langsmith allow us to log "chain"
@ -143,7 +150,7 @@ class LangsmithLogger(CustomBatchLogger):
"start_time": payload["startTime"],
"end_time": payload["endTime"],
"tags": payload["request_tags"],
"extra": metadata,
"extra": extra_metadata,
}
if payload["error_str"] is not None and payload["status"] == "failure":
@ -439,9 +446,9 @@ class LangsmithLogger(CustomBatchLogger):
return log_queue_by_credentials
def _get_sampling_rate_to_use_for_request(self, kwargs: Dict[str, Any]) -> float:
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params", None)
sampling_rate: float = self.sampling_rate
if standard_callback_dynamic_params is not None:
_sampling_rate = standard_callback_dynamic_params.get(
@ -461,9 +468,9 @@ class LangsmithLogger(CustomBatchLogger):
Otherwise, use the default credentials.
"""
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params", None)
if standard_callback_dynamic_params is not None:
credentials = self.get_credentials_from_env(
langsmith_api_key=standard_callback_dynamic_params.get(

22
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -197,12 +197,22 @@ def extract_and_raise_litellm_exception(
exception_name = exception_name.strip().replace("litellm.", "")
raised_exception_obj = getattr(litellm, exception_name, None)
if raised_exception_obj:
raise raised_exception_obj(
message=error_str,
llm_provider=custom_llm_provider,
model=model,
response=response,
)
# Try with response parameter first, fall back to without it
# Some exceptions (e.g., APIConnectionError) don't accept response param
try:
raise raised_exception_obj(
message=error_str,
llm_provider=custom_llm_provider,
model=model,
response=response,
)
except TypeError:
# Exception doesn't accept response parameter
raise raised_exception_obj(
message=error_str,
llm_provider=custom_llm_provider,
model=model,
)
def exception_type( # type: ignore # noqa: PLR0915

1
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -1533,6 +1533,7 @@ class AWSEventStreamDecoder:
)
],
id=self.response_id,
model=self.model,
usage=usage,
provider_specific_fields=model_response_provider_specific_fields,
)

1
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -236,6 +236,7 @@ class BedrockPassthroughConfig(
if len(all_translated_chunks) > 0:
model_response = stream_chunk_builder(
chunks=all_translated_chunks,
logging_obj=litellm_logging_obj,
)
return model_response
return None

50
litellm/llms/ollama/chat/transformation.py
View file

@ -190,46 +190,14 @@ class OllamaChatConfig(BaseConfig):
else:
optional_params["think"] = value in {"low", "medium", "high"}
### FUNCTION CALLING LOGIC ###
# Ollama 0.4+ supports native tool calling - pass tools directly
# and let Ollama handle model capability detection
# Fixes: https://github.com/BerriAI/litellm/issues/18922
if param == "tools":
## CHECK IF MODEL SUPPORTS TOOL CALLING ##
try:
model_info = litellm.get_model_info(
model=model, custom_llm_provider="ollama"
)
if model_info.get("supports_function_calling") is True:
optional_params["tools"] = value
else:
raise Exception
except Exception:
optional_params["format"] = "json"
litellm.add_function_to_prompt = (
True # so that main.py adds the function call to the prompt
)
optional_params["functions_unsupported_model"] = value
if len(optional_params["functions_unsupported_model"]) == 1:
optional_params["function_name"] = optional_params[
"functions_unsupported_model"
][0]["function"]["name"]
optional_params["tools"] = value
if param == "functions":
## CHECK IF MODEL SUPPORTS TOOL CALLING ##
try:
model_info = litellm.get_model_info(
model=model, custom_llm_provider="ollama"
)
if model_info.get("supports_function_calling") is True:
optional_params["tools"] = value
else:
raise Exception
except Exception:
optional_params["format"] = "json"
litellm.add_function_to_prompt = (
True # so that main.py adds the function call to the prompt
)
optional_params["functions_unsupported_model"] = (
non_default_params.get("functions")
)
optional_params["tools"] = value
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
non_default_params.pop("functions", None) # causes ollama requests to hang
return optional_params
@ -431,6 +399,10 @@ class OllamaChatConfig(BaseConfig):
_message = litellm.Message(**response_json_message)
model_response.choices[0].message = _message # type: ignore
# Set finish_reason to "tool_calls" when tool_calls are present
# Fixes: https://github.com/BerriAI/litellm/issues/18922
if _message.tool_calls:
model_response.choices[0].finish_reason = "tool_calls"
model_response.created = int(time.time())
model_response.model = "ollama_chat/" + model
prompt_tokens = response_json.get("prompt_eval_count", litellm.token_counter(messages=messages)) # type: ignore
@ -563,6 +535,10 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
if chunk["done"] is True:
finish_reason = chunk.get("done_reason", "stop")
# Override finish_reason when tool_calls are present
# Fixes: https://github.com/BerriAI/litellm/issues/18922
if tool_calls is not None:
finish_reason = "tool_calls"
choices = [
StreamingChoices(
delta=delta,

2
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -362,7 +362,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if has_stream_ended:
# convert to model response
model_response = cast(
ModelResponse, stream_chunk_builder(chunks=responses_so_far)
ModelResponse, stream_chunk_builder(chunks=responses_so_far, logging_obj=litellm_logging_obj)
)
# run process_output_response
await self.process_output_response(

65
litellm/llms/watsonx/audio_transcription/transformation.py
View file

@ -22,7 +22,7 @@ from ...base_llm.audio_transcription.transformation import (
from ...openai.transcriptions.whisper_transformation import (
OpenAIWhisperAudioTranscriptionConfig,
)
from ..common_utils import IBMWatsonXMixin, _get_api_params
from ..common_utils import IBMWatsonXMixin
class IBMWatsonXAudioTranscriptionConfig(
@ -48,7 +48,7 @@ class IBMWatsonXAudioTranscriptionConfig(
) -> Dict:
"""
Validate environment for audio transcription.
Removes Content-Type header so httpx can set multipart/form-data automatically.
"""
result = IBMWatsonXMixin.validate_environment(
@ -88,31 +88,37 @@ class IBMWatsonXAudioTranscriptionConfig(
) -> AudioTranscriptionRequestData:
"""
Transform the audio transcription request for WatsonX.
WatsonX expects multipart/form-data with:
- file: the audio file
- model: the model name (without watsonx/ prefix)
- project_id: the project ID (as form field, not query param)
- space_id: the space ID (as form field, not query param)
- other optional params
"""
# Use common utility to process the audio file
processed_audio = process_audio_file(audio_file)
# Get API params to extract project_id
api_params = _get_api_params(params=optional_params.copy())
project_id = optional_params.get("project_id") or optional_params.get(
"watsonx_project"
)
space_id = optional_params.get("space_id")
# api_params = _get_api_params(params=optional_params, model=model)
# Initialize form data with required fields
form_data: WatsonXAudioTranscriptionRequestBody = {
"model": model,
"project_id": api_params.get("project_id", ""),
}
form_data: WatsonXAudioTranscriptionRequestBody = {"model": model}
# Only add project_id or space_id if they were explicitly provided by the user
if project_id:
form_data["project_id"] = project_id
elif space_id:
form_data["space_id"] = space_id
# Add supported OpenAI params to form data
supported_params = self.get_supported_openai_params(model)
for key, value in optional_params.items():
if key in supported_params and value is not None:
form_data[key] = value # type: ignore
# Prepare files dict with the audio file
files = {
"file": (
@ -121,10 +127,10 @@ class IBMWatsonXAudioTranscriptionConfig(
processed_audio.content_type,
)
}
# Convert TypedDict to regular dict for AudioTranscriptionRequestData
form_data_dict: Dict[str, Any] = dict(form_data)
return AudioTranscriptionRequestData(data=form_data_dict, files=files)
def get_complete_url(
@ -140,8 +146,8 @@ class IBMWatsonXAudioTranscriptionConfig(
Construct the complete URL for WatsonX audio transcription.
URL format: {api_base}/ml/v1/audio/transcriptions?version={version}
Note: project_id is sent as form data, not as a query parameter
Note: project_id or space_id is sent as form data, not as a query parameter
"""
# Get base URL
url = self._get_base_url(api_base=api_base)
@ -151,9 +157,10 @@ class IBMWatsonXAudioTranscriptionConfig(
url = f"{url}/ml/v1/audio/transcriptions"
# Add version parameter (only version in query string, not project_id)
api_version = optional_params.get(
"api_version", None
) or litellm.WATSONX_DEFAULT_API_VERSION
api_version = (
optional_params.get("api_version", None)
or litellm.WATSONX_DEFAULT_API_VERSION
)
url = f"{url}?version={api_version}"
return url
@ -164,7 +171,7 @@ class IBMWatsonXAudioTranscriptionConfig(
) -> TranscriptionResponse:
"""
Transform the audio transcription response from WatsonX.
WatsonX may include a 'model' field in the response, which needs to be
removed before creating the TranscriptionResponse object.
"""
@ -179,26 +186,30 @@ class IBMWatsonXAudioTranscriptionConfig(
# TranscriptionResponse only accepts 'text' and 'usage' in __init__()
text = raw_response_json.get("text")
usage = raw_response_json.get("usage")
# Create response with only valid fields
response_kwargs = {}
if text is not None:
response_kwargs["text"] = text
if usage is not None:
response_kwargs["usage"] = usage
if not response_kwargs:
raise ValueError(
"Invalid response format. Received response does not match the expected format. Got: ",
raw_response_json,
)
response = TranscriptionResponse(**response_kwargs)
# Add other fields using dictionary-style assignment (like duration, task, etc.)
# Skip fields that TranscriptionResponse doesn't accept in __init__()
for key, value in raw_response_json.items():
if key not in ["text", "usage", "model"]: # text/usage already set, model should be excluded
if key not in [
"text",
"usage",
"model",
]: # text/usage already set, model should be excluded
response[key] = value
return response

2
litellm/llms/watsonx/chat/handler.py
View file

@ -40,7 +40,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler):
streaming_decoder: Optional[CustomStreamingDecoder] = None,
fake_stream: bool = False,
):
api_params = _get_api_params(params=optional_params)
api_params = _get_api_params(params=optional_params, model=model)
## UPDATE HEADERS
headers = watsonx_chat_transformation.validate_environment(

13
litellm/llms/watsonx/chat/transformation.py
View file

@ -10,7 +10,6 @@ from litellm import verbose_logger
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.watsonx import (
WatsonXAIEndpoint,
WatsonXAPIParams,
WatsonXModelPattern,
)
@ -115,18 +114,6 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
)
return url
def _prepare_payload(self, model: str, api_params: WatsonXAPIParams) -> dict:
"""
Prepare payload for deployment models.
Deployment models cannot have 'model_id' or 'model' in the request body.
"""
payload: dict = {}
payload["model_id"] = None if model.startswith("deployment/") else model
payload["project_id"] = (
None if model.startswith("deployment/") else api_params["project_id"]
)
return payload
@staticmethod
def _apply_prompt_template_core(
model: str, messages: List[Dict[str, str]], hf_template_fn

39
litellm/llms/watsonx/common_utils.py
View file

@ -80,9 +80,7 @@ def _generate_watsonx_token(api_key: Optional[str], token: Optional[str]) -> str
return token
def _get_api_params(
params: dict,
) -> WatsonXAPIParams:
def _get_api_params(params: dict, model: Optional[str] = None) -> WatsonXAPIParams:
"""
Find watsonx.ai credentials in the params or environment variables and return the headers for authentication.
"""
@ -118,10 +116,15 @@ def _get_api_params(
or get_secret_str("SPACE_ID")
)
if project_id is None:
if (
project_id is None
and space_id is None
and model is not None
and not model.startswith("deployment/")
):
raise WatsonXAIError(
status_code=401,
message="Error: Watsonx project_id not set. Set WX_PROJECT_ID in environment variables or pass in as a parameter.",
message="Error: Watsonx project_id and space_id not set. Set WX_PROJECT_ID or WX_SPACE_ID in environment variables or pass in as a parameter.",
)
return WatsonXAPIParams(
@ -146,7 +149,9 @@ async def _aconvert_watsonx_messages_core(
model_prompt_dict = custom_prompt_dict[model]
return ptf.custom_prompt(
messages=messages,
role_dict=model_prompt_dict.get("role_dict", model_prompt_dict.get("roles")),
role_dict=model_prompt_dict.get(
"role_dict", model_prompt_dict.get("roles")
),
initial_prompt_value=model_prompt_dict.get("initial_prompt_value", ""),
final_prompt_value=model_prompt_dict.get("final_prompt_value", ""),
bos_token=model_prompt_dict.get("bos_token", ""),
@ -180,7 +185,9 @@ def _convert_watsonx_messages_core(
model_prompt_dict = custom_prompt_dict[model]
return ptf.custom_prompt(
messages=messages,
role_dict=model_prompt_dict.get("role_dict", model_prompt_dict.get("roles")),
role_dict=model_prompt_dict.get(
"role_dict", model_prompt_dict.get("roles")
),
initial_prompt_value=model_prompt_dict.get("initial_prompt_value", ""),
final_prompt_value=model_prompt_dict.get("final_prompt_value", ""),
bos_token=model_prompt_dict.get("bos_token", ""),
@ -200,7 +207,10 @@ def _convert_watsonx_messages_core(
async def aconvert_watsonx_messages_to_prompt(
model: str, messages: List[AllMessageValues], provider: str, custom_prompt_dict: Dict
model: str,
messages: List[AllMessageValues],
provider: str,
custom_prompt_dict: Dict,
) -> str:
"""Async version of convert_watsonx_messages_to_prompt"""
from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
@ -215,7 +225,10 @@ async def aconvert_watsonx_messages_to_prompt(
def convert_watsonx_messages_to_prompt(
model: str, messages: List[AllMessageValues], provider: str, custom_prompt_dict: Dict
model: str,
messages: List[AllMessageValues],
provider: str,
custom_prompt_dict: Dict,
) -> str:
"""Sync version of convert_watsonx_messages_to_prompt"""
from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
@ -254,7 +267,8 @@ class IBMWatsonXMixin:
)
zen_api_key = cast(
Optional[str],
optional_params.pop("zen_api_key", None) or get_secret_str("WATSONX_ZENAPIKEY"),
optional_params.pop("zen_api_key", None)
or get_secret_str("WATSONX_ZENAPIKEY"),
)
if token:
headers["Authorization"] = f"Bearer {token}"
@ -360,5 +374,8 @@ class IBMWatsonXMixin:
{}
) # Deployment models do not support 'space_id' or 'project_id' in their payload
payload["model_id"] = model
payload["project_id"] = api_params["project_id"]
if api_params["project_id"] is not None:
payload["project_id"] = api_params["project_id"]
else:
payload["space_id"] = api_params["space_id"]
return payload

50
litellm/llms/watsonx/completion/transformation.py
View file

@ -228,13 +228,17 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig):
"us-south",
]
def _build_request_payload(self, model: str, prompt: str, optional_params: Dict) -> Dict:
def _build_request_payload(
self, model: str, prompt: str, optional_params: Dict
) -> Dict:
"""Shared logic to build request payload"""
extra_body_params = optional_params.pop("extra_body", {})
optional_params.update(extra_body_params)
watsonx_api_params = _get_api_params(params=optional_params)
watsonx_auth_payload = self._prepare_payload(model=model, api_params=watsonx_api_params)
watsonx_api_params = _get_api_params(params=optional_params, model=model)
watsonx_auth_payload = self._prepare_payload(
model=model, api_params=watsonx_api_params
)
return {
"input": prompt,
"moderations": optional_params.pop("moderations", {}),
@ -242,21 +246,43 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig):
**watsonx_auth_payload,
}
async def atransform_request(self, model: str, messages: List[AllMessageValues], optional_params: Dict, litellm_params: Dict, headers: Dict) -> Dict:
async def atransform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: Dict,
litellm_params: Dict,
headers: Dict,
) -> Dict:
"""Async version of transform_request"""
from litellm.llms.watsonx.common_utils import (
aconvert_watsonx_messages_to_prompt,
)
provider = model.split("/")[0]
prompt = await aconvert_watsonx_messages_to_prompt(model=model, messages=messages, provider=provider, custom_prompt_dict={})
return self._build_request_payload(model=model, prompt=prompt, optional_params=optional_params)
def transform_request(self, model: str, messages: List[AllMessageValues], optional_params: Dict, litellm_params: Dict, headers: Dict) -> Dict:
prompt = await aconvert_watsonx_messages_to_prompt(
model=model, messages=messages, provider=provider, custom_prompt_dict={}
)
return self._build_request_payload(
model=model, prompt=prompt, optional_params=optional_params
)
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: Dict,
litellm_params: Dict,
headers: Dict,
) -> Dict:
"""Sync version of transform_request"""
provider = model.split("/")[0]
prompt = convert_watsonx_messages_to_prompt(model=model, messages=messages, provider=provider, custom_prompt_dict={})
return self._build_request_payload(model=model, prompt=prompt, optional_params=optional_params)
prompt = convert_watsonx_messages_to_prompt(
model=model, messages=messages, provider=provider, custom_prompt_dict={}
)
return self._build_request_payload(
model=model, prompt=prompt, optional_params=optional_params
)
def transform_response(
self,

2
litellm/llms/watsonx/embed/transformation.py
View file

@ -37,7 +37,7 @@ class IBMWatsonXEmbeddingConfig(IBMWatsonXMixin, BaseEmbeddingConfig):
optional_params: dict,
headers: dict,
) -> dict:
watsonx_api_params = _get_api_params(params=optional_params)
watsonx_api_params = _get_api_params(params=optional_params, model=model)
watsonx_auth_payload = self._prepare_payload(
model=model,
api_params=watsonx_api_params,

14
litellm/proxy/guardrails/guardrail_registry.py
View file

@ -526,10 +526,24 @@ class InMemoryGuardrailHandler:
)
default_on = litellm_params.default_on
# Extract additional params from litellm_params to pass to custom guardrail
# This matches the behavior of other guardrail initializers (e.g., initialize_lakera)
# and aligns with the documented behavior for custom guardrails
if hasattr(litellm_params, "model_dump"):
extra_params = litellm_params.model_dump(exclude_none=True)
else:
extra_params = dict(litellm_params) if litellm_params else {}
# Remove params that are handled explicitly or are internal
for key in ["guardrail", "mode", "default_on"]:
extra_params.pop(key, None)
_guardrail_callback = _guardrail_class(
guardrail_name=guardrail["guardrail_name"],
event_hook=mode,
default_on=default_on,
**extra_params,
)
litellm.logging_callback_manager.add_litellm_callback(_guardrail_callback) # type: ignore

2
litellm/proxy/prompts/prompt_endpoints.py
View file

@ -851,7 +851,7 @@ async def delete_prompt(
)
# Delete the prompt from the database
await prisma_client.db.litellm_prompttable.delete(
await prisma_client.db.litellm_prompttable.delete_many(
where={"prompt_id": prompt_id}
)

12
litellm/proxy/proxy_server.py
View file

@ -887,6 +887,10 @@ def get_openapi_schema():
from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec
openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema)
# Fix Swagger UI execute path error when server_root_path is set
if server_root_path:
openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}]
app.openapi_schema = openapi_schema
return app.openapi_schema
@ -909,6 +913,10 @@ def custom_openapi():
from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec
openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema)
# Fix Swagger UI execute path error when server_root_path is set
if server_root_path:
openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}]
app.openapi_schema = openapi_schema
return app.openapi_schema
@ -7086,8 +7094,8 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False)
f"Provider token counting failed ({result.status_code}): {result.error_message}. "
"Falling back to local tokenizer."
)
else:
# Success - return the result
elif result is not None:
# Success - return the result (only if not None)
return result
# Check if token counter is disabled before fallback

31
litellm/router.py
View file

@ -1229,6 +1229,7 @@ class Router:
self, model: str, messages: List[Dict[str, str]], **kwargs
) -> Union[ModelResponse, CustomStreamWrapper]:
model_name = None
deployment = None
try:
# pick the one that is available (lowest TPM/RPM)
deployment = self.get_available_deployment(
@ -1291,6 +1292,9 @@ class Router:
verbose_router_logger.info(
f"litellm.completion(model={model_name})\033[31m Exception {str(e)}\033[0m"
)
# Set per-deployment num_retries on exception for retry logic
if deployment is not None:
self._set_deployment_num_retries_on_exception(e, deployment)
raise e
# fmt: off
@ -1510,7 +1514,7 @@ class Router:
return FallbackStreamWrapper(stream_with_fallbacks())
async def _acompletion(
async def _acompletion( # noqa: PLR0915
self, model: str, messages: List[Dict[str, str]], **kwargs
) -> Union[ModelResponse, CustomStreamWrapper,]:
"""
@ -1520,6 +1524,7 @@ class Router:
- in the semaphore, make a check against it's local rpm before running
"""
model_name = None
deployment = None
_timeout_debug_deployment_dict = (
{}
) # this is a temporary dict to debug timeout issues
@ -1648,6 +1653,9 @@ class Router:
"litellm_params", {}
).get("timeout", None)
e.message += f"\n\nDeployment Info: request_timeout: {deployment_request_timeout_param}\ntimeout: {deployment_timeout_param}"
# Set per-deployment num_retries on exception for retry logic
if deployment is not None:
self._set_deployment_num_retries_on_exception(e, deployment)
raise e
except Exception as e:
verbose_router_logger.info(
@ -1655,6 +1663,9 @@ class Router:
)
if model_name is not None:
self.fail_calls[model_name] += 1
# Set per-deployment num_retries on exception for retry logic
if deployment is not None:
self._set_deployment_num_retries_on_exception(e, deployment)
raise e
def _update_kwargs_before_fallbacks(
@ -1678,6 +1689,24 @@ class Router:
{"model_group": model, "model_group_alias": model_group_alias}
)
def _set_deployment_num_retries_on_exception(
self, exception: Exception, deployment: dict
) -> None:
"""
Set num_retries from deployment litellm_params on the exception.
This allows the retry logic in async_function_with_retries to use
per-deployment retry settings instead of the global setting.
"""
# Only set if exception doesn't already have num_retries
if hasattr(exception, "num_retries") and exception.num_retries is not None: # type: ignore
return
litellm_params = deployment.get("litellm_params", {})
dep_num_retries = litellm_params.get("num_retries")
if dep_num_retries is not None and isinstance(dep_num_retries, int):
exception.num_retries = dep_num_retries # type: ignore
def _update_kwargs_with_default_litellm_params(
self, kwargs: dict, metadata_variable_name: Optional[str] = "metadata"
) -> None:

12
litellm/types/llms/watsonx.py
View file

@ -5,7 +5,7 @@ from typing_extensions import NotRequired, TypedDict
class WatsonXAPIParams(TypedDict):
project_id: str
project_id: Optional[str]
space_id: Optional[str]
region_name: Optional[str]
@ -19,7 +19,7 @@ class WatsonXCredentials(TypedDict):
class WatsonXAudioTranscriptionRequestBody(TypedDict):
"""
WatsonX Audio Transcription API request body.
Follows multipart/form-data format for WatsonX Whisper models.
See: https://cloud.ibm.com/apidocs/watsonx-ai
"""
@ -27,8 +27,11 @@ class WatsonXAudioTranscriptionRequestBody(TypedDict):
model: str
"""Model name (e.g., 'whisper-large-v3-turbo')"""
project_id: str
"""WatsonX project ID (required)"""
project_id: NotRequired[str]
"""WatsonX project ID (optional)"""
space_id: NotRequired[str]
"""WatsonX space ID (optional)"""
language: NotRequired[str]
"""Language code (e.g., 'en', 'es')"""
@ -64,6 +67,7 @@ class WatsonXAIEndpoint(str, Enum):
class WatsonXModelPattern(str, Enum):
"""Model identifier patterns for WatsonX models"""
GRANITE_CHAT = "granite-chat"
IBM_MISTRAL = "ibm-mistral"
IBM_MISTRALAI = "ibm-mistralai"

43
litellm/types/utils.py
View file

@ -1250,6 +1250,14 @@ class Choices(OpenAIObject):
params["message"] = message
elif isinstance(message, dict):
params["message"] = Message(**message)
elif isinstance(message, BaseModel):
# Normalize provider/OpenAI SDK message models into LiteLLM's Message type.
dump = (
message.model_dump()
if hasattr(message, "model_dump")
else message.dict()
)
params["message"] = Message(**dump)
if logprobs is not None:
if isinstance(logprobs, dict):
params["logprobs"] = ChoiceLogprobs(**logprobs)
@ -1612,6 +1620,12 @@ class ModelResponseBase(OpenAIObject):
_response_headers: Optional[dict] = None
def model_dump(self, **kwargs):
"""Default to exclude_unset to avoid Pydantic serializer warnings for OpenAIObject-derived types."""
if "exclude_unset" not in kwargs and "exclude_none" not in kwargs:
kwargs["exclude_unset"] = True
return super().model_dump(**kwargs)
class ModelResponseStream(ModelResponseBase):
choices: List[StreamingChoices]
@ -1651,12 +1665,16 @@ class ModelResponseStream(ModelResponseBase):
else:
created = created
if (
"usage" in kwargs
and kwargs["usage"] is not None
and isinstance(kwargs["usage"], dict)
):
kwargs["usage"] = Usage(**kwargs["usage"])
if "usage" in kwargs and kwargs["usage"] is not None:
if isinstance(kwargs["usage"], dict):
kwargs["usage"] = Usage(**kwargs["usage"])
elif isinstance(kwargs["usage"], BaseModel):
dump = (
kwargs["usage"].model_dump()
if hasattr(kwargs["usage"], "model_dump")
else kwargs["usage"].dict()
)
kwargs["usage"] = Usage(**dump)
kwargs["id"] = id
kwargs["created"] = created
@ -1730,6 +1748,13 @@ class ModelResponse(ModelResponseBase):
_new_choice = choice # type: ignore
elif isinstance(choice, dict):
_new_choice = Choices(**choice) # type: ignore
elif isinstance(choice, BaseModel):
dump = (
choice.model_dump()
if hasattr(choice, "model_dump")
else choice.dict()
)
_new_choice = Choices(**dump) # type: ignore
else:
_new_choice = choice
new_choices.append(_new_choice)
@ -1748,6 +1773,11 @@ class ModelResponse(ModelResponseBase):
if usage is not None:
if isinstance(usage, dict):
usage = Usage(**usage)
elif isinstance(usage, BaseModel):
dump = (
usage.model_dump() if hasattr(usage, "model_dump") else usage.dict()
)
usage = Usage(**dump)
else:
usage = usage
elif stream is None or stream is False:
@ -3032,7 +3062,6 @@ class LlmProviders(str, Enum):
XIAOMI_MIMO = "xiaomi_mimo"
# Create a set of all provider values for quick lookup
LlmProvidersSet = {provider.value for provider in LlmProviders}

3
tests/llm_translation/test_bedrock_completion.py
View file

@ -3350,7 +3350,8 @@ async def test_bedrock_converse__streaming_passthrough(monkeypatch):
mock_callback.assert_called_once()
print(mock_callback.call_args.kwargs.keys())
assert "response_cost" in mock_callback.call_args.kwargs["kwargs"]
assert mock_callback.call_args.kwargs["kwargs"]["response_cost"] > 0
response_cost = mock_callback.call_args.kwargs["kwargs"]["response_cost"]
assert response_cost is not None and response_cost > 0
assert "standard_logging_object" in mock_callback.call_args.kwargs["kwargs"]

53
tests/llm_translation/test_watsonx.py
View file

@ -1,17 +1,14 @@
import json
import os
import sys
from datetime import datetime
from unittest.mock import AsyncMock
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import litellm
from litellm import completion, embedding
from litellm.llms.watsonx.common_utils import IBMWatsonXMixin
from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler
from unittest.mock import patch, MagicMock, AsyncMock, Mock
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from unittest.mock import patch, Mock
import pytest
from typing import Optional
@ -188,6 +185,27 @@ def test_watsonx_chat_completions_endpoint(watsonx_chat_completion_call):
assert "deployment" not in mock_post.call_args.kwargs["url"]
def test_watsonx_chat_completions_endpoint_space_id(
monkeypatch, watsonx_chat_completion_call
):
my_fake_space_id = "xxx-xxx-xxx-xxx-xxx"
monkeypatch.setenv("WATSONX_SPACE_ID", my_fake_space_id)
monkeypatch.delenv("WATSONX_PROJECT_ID", raising=False)
model = "watsonx/another-model"
messages = [{"role": "user", "content": "Test message"}]
mock_post, _ = watsonx_chat_completion_call(model=model, messages=messages)
assert mock_post.call_count == 1
assert "deployment" not in mock_post.call_args.kwargs["url"]
json_data = json.loads(mock_post.call_args.kwargs["data"])
assert my_fake_space_id == json_data["space_id"]
assert not json_data.get("project_id")
@pytest.mark.parametrize(
"model",
[
@ -209,6 +227,27 @@ def test_watsonx_deployment_space_id(monkeypatch, watsonx_chat_completion_call,
assert my_fake_space_id not in json_data
@pytest.mark.parametrize(
"model",
[
"watsonx/deployment/<xxxx.xxx.xxx.xxxx>",
"watsonx_text/deployment/<xxxx.xxx.xxx.xxxx>",
],
)
def test_watsonx_deployment(watsonx_chat_completion_call, model):
messages = [{"content": "Hello, how are you?", "role": "user"}]
mock_post, _ = watsonx_chat_completion_call(
model=model,
messages=messages,
)
assert mock_post.call_count == 1
json_data = json.loads(mock_post.call_args.kwargs["data"])
# nor space_id or project_id is required by wx.ai API when inferencing deployment
assert "project_id" not in json_data and "space_id" not in json_data
def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_call):
my_fake_space_id = "xxx-xxx-xxx-xxx-xxx"
monkeypatch.setenv("WATSONX_SPACE_ID", my_fake_space_id)
@ -217,4 +256,6 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca
assert mock_post.call_count == 1
json_data = json.loads(mock_post.call_args.kwargs["data"])
assert my_fake_space_id not in json_data
# nor space_id or project_id is required by wx.ai API when inferencing deployment
assert "project_id" not in json_data and "space_id" not in json_data

1
tests/logging_callback_tests/test_langsmith_unit_test.py
View file

@ -392,7 +392,6 @@ async def test_langsmith_key_based_logging():
"role": "assistant",
"tool_calls": None,
"function_call": None,
"provider_specific_fields": None,
},
}
],

77
tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
View file

@ -12,6 +12,7 @@ sys.path.insert(
from litellm.litellm_core_utils.exception_mapping_utils import (
ExceptionCheckers,
exception_type,
extract_and_raise_litellm_exception,
)
# Test cases for is_error_str_context_window_exceeded
@ -269,3 +270,79 @@ def test_vertex_ai_rate_limit_error_mapping(error_message, should_raise_rate_lim
original_exception=original_exception,
custom_llm_provider=custom_llm_provider,
)
class TestExtractAndRaiseLitellmException:
"""Tests for extract_and_raise_litellm_exception function"""
def test_extract_and_raise_api_connection_error_without_response(self):
"""
Test that APIConnectionError can be raised without response parameter.
This is a regression test for the bug where extract_and_raise_litellm_exception
would fail with TypeError when trying to raise APIConnectionError with a
response parameter, since APIConnectionError doesn't accept that parameter.
Relevant Issue: https://github.com/BerriAI/litellm/issues/XXXXX
"""
error_str = "litellm.APIConnectionError: GeminiException - some error message"
with pytest.raises(litellm.APIConnectionError) as excinfo:
extract_and_raise_litellm_exception(
response=None,
error_str=error_str,
model="gemini/gemini-3-pro-preview",
custom_llm_provider="gemini",
)
assert "APIConnectionError" in str(excinfo.value)
def test_extract_and_raise_bad_request_error_with_response(self):
"""
Test that BadRequestError can be raised with response parameter.
BadRequestError does accept the response parameter, so this should work.
"""
error_str = "litellm.BadRequestError: Invalid request format"
with pytest.raises(litellm.BadRequestError) as excinfo:
extract_and_raise_litellm_exception(
response=None,
error_str=error_str,
model="gpt-4",
custom_llm_provider="openai",
)
assert "BadRequestError" in str(excinfo.value)
def test_extract_and_raise_context_window_exceeded_error(self):
"""
Test that ContextWindowExceededError can be raised.
"""
error_str = "litellm.ContextWindowExceededError: Token limit exceeded"
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
extract_and_raise_litellm_exception(
response=None,
error_str=error_str,
model="gpt-4",
custom_llm_provider="openai",
)
assert "ContextWindowExceededError" in str(excinfo.value)
def test_no_exception_raised_for_non_litellm_error(self):
"""
Test that no exception is raised for non-litellm error strings.
"""
error_str = "Some generic error that is not a litellm exception"
# Should not raise any exception
result = extract_and_raise_litellm_exception(
response=None,
error_str=error_str,
model="gpt-4",
custom_llm_provider="openai",
)
assert result is None

150
tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py
View file

@ -323,3 +323,153 @@ class TestOllamaChatConfigResponseFormat:
# and the code checks "if images is not None", an empty list will still be set
assert "images" in result["messages"][0]
assert result["messages"][0]["images"] == []
class TestOllamaToolCalling:
"""Tests for Ollama tool calling fixes.
Issue: https://github.com/BerriAI/litellm/issues/18922
"""
def test_tools_passed_directly_without_capability_check(self):
"""Test that tools are passed directly to Ollama without model capability checks.
Previously, the code called litellm.get_model_info() which could fail
when Ollama runs on a remote server, causing a broken fallback.
Now tools are passed directly - Ollama 0.4+ handles capability detection.
"""
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather",
"parameters": {"type": "object", "properties": {}},
},
}
]
optional_params = get_optional_params(
model="ollama_chat/qwen3:14b",
tools=tools,
custom_llm_provider="ollama_chat",
)
# Tools should be passed through directly
assert "tools" in optional_params
assert optional_params["tools"] == tools
# Should NOT trigger the broken fallback
assert "functions_unsupported_model" not in optional_params
assert "format" not in optional_params or optional_params.get("format") != "json"
def test_finish_reason_tool_calls_non_streaming(self):
"""Test that finish_reason is set to 'tool_calls' when tool_calls present.
Previously, finish_reason was hardcoded to 'stop' even when tool_calls
were in the response, causing clients to ignore the tool calls.
"""
import json
from unittest.mock import MagicMock
import litellm
from litellm.types.utils import Choices, Message, ModelResponse
config = OllamaChatConfig()
# Simulated Ollama response with tool_calls
ollama_response = {
"model": "qwen3:14b",
"created_at": "2025-01-11T00:00:00.000000Z",
"message": {
"role": "assistant",
"content": "",
"tool_calls": [
{
"function": {
"name": "get_weather",
"arguments": {"location": "Tokyo"},
}
}
],
},
"done": True,
"prompt_eval_count": 100,
"eval_count": 50,
}
mock_response = MagicMock()
mock_response.json.return_value = ollama_response
mock_response.text = json.dumps(ollama_response)
mock_logging = MagicMock()
model_response = ModelResponse()
model_response.choices = [Choices(message=Message(content=""), index=0)]
result = config.transform_response(
model="qwen3:14b",
raw_response=mock_response,
model_response=model_response,
logging_obj=mock_logging,
request_data={},
messages=[{"role": "user", "content": "Weather?"}],
optional_params={},
litellm_params={},
encoding=None,
api_key=None,
json_mode=False,
)
# finish_reason should be "tool_calls", not "stop"
assert result.choices[0].finish_reason == "tool_calls"
assert result.choices[0].message.tool_calls is not None
def test_finish_reason_stop_when_no_tool_calls(self):
"""Test that finish_reason remains 'stop' when no tool_calls present."""
import json
from unittest.mock import MagicMock
import litellm
from litellm.types.utils import Choices, Message, ModelResponse
config = OllamaChatConfig()
# Simulated Ollama response without tool_calls
ollama_response = {
"model": "qwen3:14b",
"created_at": "2025-01-11T00:00:00.000000Z",
"message": {
"role": "assistant",
"content": "Hello! How can I help you?",
},
"done": True,
"prompt_eval_count": 100,
"eval_count": 50,
}
mock_response = MagicMock()
mock_response.json.return_value = ollama_response
mock_response.text = json.dumps(ollama_response)
mock_logging = MagicMock()
model_response = ModelResponse()
model_response.choices = [Choices(message=Message(content=""), index=0)]
result = config.transform_response(
model="qwen3:14b",
raw_response=mock_response,
model_response=model_response,
logging_obj=mock_logging,
request_data={},
messages=[{"role": "user", "content": "Hello"}],
optional_params={},
litellm_params={},
encoding=None,
api_key=None,
json_mode=False,
)
# finish_reason should be "stop" (default behavior)
assert result.choices[0].finish_reason == "stop"
assert result.choices[0].message.tool_calls is None

138
tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py
View file

@ -35,7 +35,7 @@ class TestWatsonXAudioTranscription:
captured_request["headers"] = kwargs.get("headers", {})
captured_request["data"] = kwargs.get("data", {})
captured_request["files"] = kwargs.get("files", {})
mock_response = MagicMock()
mock_response.json.return_value = {
"text": "test transcription",
@ -44,7 +44,10 @@ class TestWatsonXAudioTranscription:
mock_response.status_code = 200
return mock_response
with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=mock_post):
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new=mock_post,
):
try:
await litellm.atranscription(
model="watsonx/whisper-large-v3-turbo",
@ -65,14 +68,16 @@ class TestWatsonXAudioTranscription:
# Validate headers contain WatsonX auth
assert "Authorization" in captured_request["headers"]
assert "Bearer test-bearer-token" in captured_request["headers"]["Authorization"]
assert (
"Bearer test-bearer-token" in captured_request["headers"]["Authorization"]
)
# Validate Content-Type is NOT set (httpx sets multipart/form-data automatically)
assert "Content-Type" not in captured_request["headers"]
# Validate project_id is in form data, not URL
assert captured_request["data"].get("project_id") == "test-project-123"
# Validate file is in files dict
assert "file" in captured_request["files"]
@ -80,7 +85,7 @@ class TestWatsonXAudioTranscription:
async def test_watsonx_transcription_request_body(self):
"""
Test that litellm.transcription sends correct request body for WatsonX.
Validates that:
- Request uses multipart/form-data (data + files)
- Model name has watsonx/ prefix removed
@ -93,7 +98,7 @@ class TestWatsonXAudioTranscription:
async def mock_post(*args, **kwargs):
captured_request["data"] = kwargs.get("data", {})
captured_request["files"] = kwargs.get("files", {})
mock_response = MagicMock()
mock_response.json.return_value = {
"text": "test transcription",
@ -102,7 +107,10 @@ class TestWatsonXAudioTranscription:
mock_response.status_code = 200
return mock_response
with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=mock_post):
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new=mock_post,
):
try:
await litellm.atranscription(
model="watsonx/whisper-large-v3-turbo",
@ -122,29 +130,31 @@ class TestWatsonXAudioTranscription:
print("JSON DUMPS captured_request:")
print(json.dumps(captured_request, indent=4, default=str))
# Model name should NOT have watsonx/ prefix
assert data.get("model") == "whisper-large-v3-turbo"
# project_id should be in form data
assert data.get("project_id") == "test-project-123"
# OpenAI params should be in form data
assert data.get("language") == "en"
assert data.get("temperature") == 0.5
# response_format should NOT be set by default - only send what user specifies
assert "response_format" not in data
# Validate file is in files dict (multipart/form-data)
files = captured_request.get("files", {})
assert "file" in files
assert isinstance(files["file"], tuple) # Should be (filename, content, content_type)
assert isinstance(
files["file"], tuple
) # Should be (filename, content, content_type)
@pytest.mark.asyncio
async def test_watsonx_transcription_only_user_params_sent(self):
async def test_watsonx_transcription_only_user_params_sent_with_project_id(self):
"""
Test that only user-specified params are sent in request body to WatsonX.
LiteLLM should NOT add extra params like response_format if user didn't specify them.
"""
captured_request = {}
@ -152,7 +162,7 @@ class TestWatsonXAudioTranscription:
async def mock_post(*args, **kwargs):
captured_request["data"] = kwargs.get("data", {})
captured_request["files"] = kwargs.get("files", {})
mock_response = MagicMock()
mock_response.json.return_value = {
"text": "test transcription",
@ -161,7 +171,10 @@ class TestWatsonXAudioTranscription:
mock_response.status_code = 200
return mock_response
with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=mock_post):
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new=mock_post,
):
try:
# Minimal request - only required params
await litellm.atranscription(
@ -176,20 +189,81 @@ class TestWatsonXAudioTranscription:
pass # We just want to capture the request
data = captured_request.get("data", {})
# These are the ONLY keys that should be in data
expected_keys = {"model", "project_id"}
actual_keys = set(data.keys())
assert actual_keys == expected_keys, (
f"Request body should only contain {expected_keys}, "
f"but got {actual_keys}. "
f"Extra keys: {actual_keys - expected_keys}"
)
# Specifically verify response_format is NOT added
assert "response_format" not in data, "response_format should NOT be added by default"
assert (
"response_format" not in data
), "response_format should NOT be added by default"
# Verify file is sent separately
files = captured_request.get("files", {})
assert "file" in files
@pytest.mark.asyncio
async def test_watsonx_transcription_only_user_params_sent_with_space_id(self):
"""
Test that only user-specified params are sent in request body to WatsonX.
LiteLLM should NOT add extra params like response_format if user didn't specify them.
"""
captured_request = {}
async def mock_post(*args, **kwargs):
captured_request["data"] = kwargs.get("data", {})
captured_request["files"] = kwargs.get("files", {})
mock_response = MagicMock()
mock_response.json.return_value = {
"text": "test transcription",
"duration": 1.0,
}
mock_response.status_code = 200
return mock_response
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new=mock_post,
):
try:
# Minimal request - only required params
await litellm.atranscription(
model="watsonx/whisper-large-v3-turbo",
file=b"fake_audio_data",
api_base="https://us-south.ml.cloud.ibm.com",
api_key="test-api-key",
space_id="test-space_id-123",
token="test-bearer-token",
)
except Exception:
pass # We just want to capture the request
data = captured_request.get("data", {})
# These are the ONLY keys that should be in data
expected_keys = {"model", "space_id"}
actual_keys = set(data.keys())
assert actual_keys == expected_keys, (
f"Request body should only contain {expected_keys}, "
f"but got {actual_keys}. "
f"Extra keys: {actual_keys - expected_keys}"
)
# Specifically verify response_format is NOT added
assert (
"response_format" not in data
), "response_format should NOT be added by default"
# Verify file is sent separately
files = captured_request.get("files", {})
assert "file" in files
@ -198,7 +272,7 @@ class TestWatsonXAudioTranscription:
"""
Test that transform_audio_transcription_response removes the 'model' field
from WatsonX response before creating TranscriptionResponse.
This test ensures that when WatsonX returns a response with a 'model' field,
it is removed before creating the TranscriptionResponse object, since
TranscriptionResponse doesn't accept a 'model' parameter.
@ -219,13 +293,13 @@ class TestWatsonXAudioTranscription:
# Verify the result is a TranscriptionResponse
assert isinstance(result, TranscriptionResponse)
# Verify the text is correct
assert result.text == "Hello, this is a test transcription."
# Verify duration is set via dictionary assignment
assert result["duration"] == 5.5
# Verify the model field is NOT in the serialized result
# Check via model_dump() or dict() to ensure it's not in the output
try:
@ -233,7 +307,7 @@ class TestWatsonXAudioTranscription:
except AttributeError:
# Fallback for pydantic v1
result_dict = result.dict()
# The 'model' field should not be in the result
assert "model" not in result_dict, "Model field should be removed from response"
@ -250,15 +324,17 @@ class TestWatsonXAudioTranscription:
"text": "Hello, this is a test transcription.",
"duration": 5.5,
}
mock_response.text = '{"text": "Hello, this is a test transcription.", "duration": 5.5}'
mock_response.text = (
'{"text": "Hello, this is a test transcription.", "duration": 5.5}'
)
result = handler.transform_audio_transcription_response(mock_response)
# Verify the result is a TranscriptionResponse
assert isinstance(result, TranscriptionResponse)
# Verify the text is correct
assert result.text == "Hello, this is a test transcription."
# Verify duration is set via dictionary assignment
assert result["duration"] == 5.5

4
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -201,6 +201,10 @@ ignored_keys = [
"metadata.usage_object",
"metadata.cold_storage_object_key",
"metadata.additional_usage_values.prompt_tokens_details.cache_creation_tokens",
"metadata.additional_usage_values.completion_tokens_details",
"metadata.additional_usage_values.prompt_tokens_details",
"metadata.additional_usage_values.cache_creation_input_tokens",
"metadata.additional_usage_values.cache_read_input_tokens",
"metadata.litellm_overhead_time_ms",
"metadata.cost_breakdown",
]

41
tests/test_litellm/proxy/test_swagger_chat_completions.py
View file

@ -307,4 +307,43 @@ class TestSwaggerChatCompletions:
# Verify required fields are present in test request
required_fields = schema_def.get("required", [])
for required_field in required_fields:
assert required_field in test_request, f"Required field '{required_field}' should be in test request"
assert required_field in test_request, f"Required field '{required_field}' should be in test request"
def test_openapi_schema_servers_url_with_root_path(self):
"""
Test that OpenAPI schema includes correct servers URL when server_root_path is set.
This ensures Swagger UI works correctly with reverse proxies and subpath deployments.
"""
from unittest.mock import patch
from litellm.proxy.proxy_server import get_openapi_schema, custom_openapi, app
# Test cases: (server_root_path, expected_servers_url)
# Note: empty string is falsy in Python, so servers won't be set
test_cases = [
("/litellm", "/litellm"),
("/litellm/", "/litellm"), # trailing slash should be removed
("litellm", "/litellm"), # missing leading slash should be added
("/api/v1", "/api/v1"),
]
for root_path, expected_url in test_cases:
# Clear cached schema
app.openapi_schema = None
with patch("litellm.proxy.proxy_server.server_root_path", root_path):
# Test get_openapi_schema
schema = get_openapi_schema()
# Should have servers field with correct URL
assert "servers" in schema, f"servers field should exist when server_root_path={root_path}"
assert schema["servers"][0]["url"] == expected_url, \
f"Expected servers URL '{expected_url}', got '{schema['servers'][0]['url']}' for root_path '{root_path}'"
# Test custom_openapi as well
app.openapi_schema = None
with patch("litellm.proxy.proxy_server.server_root_path", root_path):
schema = custom_openapi()
assert "servers" in schema, f"servers field should exist in custom_openapi when server_root_path={root_path}"
assert schema["servers"][0]["url"] == expected_url, \
f"Expected servers URL '{expected_url}' in custom_openapi, got '{schema['servers'][0]['url']}'"

61
tests/test_litellm/test_model_response_normalization.py Normal file
View file

@ -0,0 +1,61 @@
import warnings
import pytest
from litellm.types.utils import Choices, Message, ModelResponse
def test_modelresponse_normalizes_openai_base_models() -> None:
# OpenAI SDK returns Pydantic BaseModel objects for message/choice.
# LiteLLM should normalize these into its own internal `Message` / `Choices` types.
from openai.types.chat.chat_completion import Choice as OpenAIChoice
from openai.types.chat.chat_completion_message import ChatCompletionMessage
message = ChatCompletionMessage(role="assistant", content="hi")
choice = OpenAIChoice(finish_reason="stop", index=0, message=message, logprobs=None)
with warnings.catch_warnings(record=True) as captured:
warnings.simplefilter("always")
response = ModelResponse(model="gpt-4o-mini", choices=[choice])
_ = response.model_dump()
assert isinstance(response.choices[0], Choices)
assert isinstance(response.choices[0].message, Message)
assert not any(
"Pydantic serializer warnings" in str(w.message)
for w in captured
if isinstance(w.message, Warning)
)
def test_modelresponse_serialization_avoids_pydantic_warnings() -> None:
pytest.importorskip("openai")
from openai.types.chat import ChatCompletion as OpenAIChatCompletion
openai_completion = OpenAIChatCompletion(
id="test-1",
created=1719868600,
model="gpt-4o-mini",
object="chat.completion",
choices=[
{
"index": 0,
"finish_reason": "stop",
"message": {"role": "assistant", "content": "hi"},
"logprobs": None,
}
],
usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
)
with warnings.catch_warnings(record=True) as captured:
warnings.simplefilter("always")
response = ModelResponse(**openai_completion.model_dump())
_ = response.model_dump(exclude_none=True)
assert not any(
"PydanticSerializationUnexpectedValue" in str(w.message)
or "Pydantic serializer warnings" in str(w.message)
for w in captured
)

157
tests/test_litellm/test_per_deployment_num_retries.py Normal file
View file

@ -0,0 +1,157 @@
"""
Unit tests for per-deployment num_retries in litellm_params
GitHub Issue: #18968 - Per-deployment max_retries/num_retries in litellm_params is not used in retry logic
"""
import pytest
from unittest.mock import MagicMock, patch
from litellm import Router
class TestPerDeploymentNumRetries:
"""Test that per-deployment num_retries in litellm_params is correctly used."""
def test_set_deployment_num_retries_on_exception(self):
"""
Test that _set_deployment_num_retries_on_exception sets num_retries
on the exception from the deployment's litellm_params.
"""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "openai/gpt-4",
"api_key": "test-key",
"num_retries": 5, # Per-deployment setting
},
},
],
num_retries=1, # Global setting
)
deployment = router.model_list[0]
# Create a mock exception without num_retries
class MockException(Exception):
pass
exc = MockException("test error")
assert not hasattr(exc, "num_retries") or exc.num_retries is None
# Call the helper
router._set_deployment_num_retries_on_exception(exc, deployment)
# Verify num_retries was set from deployment
assert exc.num_retries == 5
def test_set_deployment_num_retries_does_not_override_existing(self):
"""
Test that _set_deployment_num_retries_on_exception does NOT override
if exception already has num_retries set.
"""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "openai/gpt-4",
"api_key": "test-key",
"num_retries": 5,
},
},
],
num_retries=1,
)
deployment = router.model_list[0]
# Create an exception that already has num_retries
class MockException(Exception):
num_retries = 10 # Already set
exc = MockException("test error")
# Call the helper
router._set_deployment_num_retries_on_exception(exc, deployment)
# Verify num_retries was NOT overridden
assert exc.num_retries == 10
def test_deployment_without_num_retries(self):
"""
Test that _set_deployment_num_retries_on_exception does nothing
if deployment has no num_retries set.
"""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "openai/gpt-4",
"api_key": "test-key",
# No num_retries set
},
},
],
num_retries=3,
)
deployment = router.model_list[0]
class MockException(Exception):
pass
exc = MockException("test error")
# Call the helper
router._set_deployment_num_retries_on_exception(exc, deployment)
# Verify num_retries was not set (deployment has no num_retries)
assert not hasattr(exc, "num_retries") or exc.num_retries is None
def test_request_level_num_retries_takes_precedence(self):
"""
Test that request-level num_retries (passed in kwargs) is still respected.
"""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "openai/gpt-4",
"api_key": "test-key",
"num_retries": 5,
},
},
],
num_retries=1,
)
# Pass num_retries in request kwargs - this should take precedence
kwargs = {"num_retries": 10}
router._update_kwargs_before_fallbacks(model="test-model", kwargs=kwargs)
assert kwargs["num_retries"] == 10 # Request-level takes precedence
def test_global_num_retries_used_when_no_deployment_setting(self):
"""
Test that global num_retries is used when deployment has no num_retries.
"""
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "openai/gpt-4",
"api_key": "test-key",
# No num_retries set
},
},
],
num_retries=7, # Global setting
)
kwargs = {}
router._update_kwargs_before_fallbacks(model="test-model", kwargs=kwargs)
assert kwargs["num_retries"] == 7 # Uses global

4
tests/test_litellm/test_router.py
View file

@ -1200,9 +1200,7 @@ async def test_acompletion_streaming_disable_fallbacks_midstream():
print("\n=== Test 1: disable_fallbacks=True with original_exception ===")
# Create an original exception to wrap
from litellm.llms.anthropic.chat.anthropic_chat_transformation import (
AnthropicError,
)
from litellm.llms.anthropic.common_utils import AnthropicError
original_error = AnthropicError(
status_code=500,

4
ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx
View file

@ -54,7 +54,7 @@ import { makeOpenAIResponsesRequest } from "../llm_calls/responses_api";
import CodeInterpreterOutput from "./CodeInterpreterOutput";
import { useCodeInterpreter } from "./useCodeInterpreter";
import { Agent, fetchAvailableAgents } from "../llm_calls/fetch_agents";
import { makeA2AStreamMessageRequest } from "../llm_calls/a2a_send_message";
import { makeA2AStreamMessageRequest, makeA2ASendMessageRequest } from "../llm_calls/a2a_send_message";
import A2AMetrics from "./A2AMetrics";
import { A2ATaskMetadata } from "./types";
import MCPEventsDisplay, { MCPEvent } from "./MCPEventsDisplay";
@ -1037,7 +1037,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
// Handle A2A agent calls (separate from model-based calls) - use streaming
if (endpointType === EndpointType.A2A_AGENTS && selectedAgent) {
await makeA2AStreamMessageRequest(
await makeA2ASendMessageRequest(
selectedAgent,
inputMessage,
(chunk, model) => updateTextUI("assistant", chunk, model),

3
uv.lock generated Normal file
View file

@ -0,0 +1,3 @@
version = 1
revision = 3
requires-python = ">=3.13"