diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index a38a29491ef..ba32dc1bf54 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -35,6 +35,7 @@ jobs: poetry run pip install "google-cloud-aiplatform>=1.38" poetry run pip install "fastapi-offline==1.7.3" poetry run pip install "python-multipart==0.0.18" + poetry run pip install "openapi-core" - name: Setup litellm-enterprise as local package run: | cd enterprise diff --git a/Makefile b/Makefile index 1614a58fc7d..0da83c363cd 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ install-proxy-dev-ci: install-test-deps: install-proxy-dev poetry run pip install "pytest-retry==1.6.3" poetry run pip install pytest-xdist + poetry run pip install openapi-core cd enterprise && poetry run pip install -e . && cd .. install-helm-unittest: @@ -100,4 +101,4 @@ test-llm-translation-single: install-test-deps @mkdir -p test-results poetry run pytest tests/llm_translation/$(FILE) \ --junitxml=test-results/junit.xml \ - -v --tb=short --maxfail=100 --timeout=300 \ No newline at end of file + -v --tb=short --maxfail=100 --timeout=300 diff --git a/enterprise/litellm_enterprise/proxy/__init__.py b/enterprise/litellm_enterprise/proxy/__init__.py new file mode 100644 index 00000000000..52b74882bc9 --- /dev/null +++ b/enterprise/litellm_enterprise/proxy/__init__.py @@ -0,0 +1 @@ +# Package marker for enterprise proxy components. diff --git a/enterprise/litellm_enterprise/proxy/common_utils/__init__.py b/enterprise/litellm_enterprise/proxy/common_utils/__init__.py new file mode 100644 index 00000000000..fe8384c8925 --- /dev/null +++ b/enterprise/litellm_enterprise/proxy/common_utils/__init__.py @@ -0,0 +1 @@ +# Package marker for enterprise proxy common utilities. diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 570b78f2927..5893f14105d 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -134,6 +134,13 @@ class LangsmithLogger(CustomBatchLogger): "metadata" ] # ensure logged metadata is json serializable + extra_metadata = dict(metadata) + requester_metadata = extra_metadata.get("requester_metadata") + if requester_metadata and isinstance(requester_metadata, dict): + for key in ("session_id", "thread_id", "conversation_id"): + if key in requester_metadata and key not in extra_metadata: + extra_metadata[key] = requester_metadata[key] + data = { "name": run_name, "run_type": "llm", # this should always be llm, since litellm always logs llm calls. Langsmith allow us to log "chain" @@ -143,7 +150,7 @@ class LangsmithLogger(CustomBatchLogger): "start_time": payload["startTime"], "end_time": payload["endTime"], "tags": payload["request_tags"], - "extra": metadata, + "extra": extra_metadata, } if payload["error_str"] is not None and payload["status"] == "failure": @@ -439,9 +446,9 @@ class LangsmithLogger(CustomBatchLogger): return log_queue_by_credentials def _get_sampling_rate_to_use_for_request(self, kwargs: Dict[str, Any]) -> float: - standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = ( - kwargs.get("standard_callback_dynamic_params", None) - ) + standard_callback_dynamic_params: Optional[ + StandardCallbackDynamicParams + ] = kwargs.get("standard_callback_dynamic_params", None) sampling_rate: float = self.sampling_rate if standard_callback_dynamic_params is not None: _sampling_rate = standard_callback_dynamic_params.get( @@ -461,9 +468,9 @@ class LangsmithLogger(CustomBatchLogger): Otherwise, use the default credentials. """ - standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = ( - kwargs.get("standard_callback_dynamic_params", None) - ) + standard_callback_dynamic_params: Optional[ + StandardCallbackDynamicParams + ] = kwargs.get("standard_callback_dynamic_params", None) if standard_callback_dynamic_params is not None: credentials = self.get_credentials_from_env( langsmith_api_key=standard_callback_dynamic_params.get( diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 1517d1e776d..107cdf39bfa 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -197,12 +197,22 @@ def extract_and_raise_litellm_exception( exception_name = exception_name.strip().replace("litellm.", "") raised_exception_obj = getattr(litellm, exception_name, None) if raised_exception_obj: - raise raised_exception_obj( - message=error_str, - llm_provider=custom_llm_provider, - model=model, - response=response, - ) + # Try with response parameter first, fall back to without it + # Some exceptions (e.g., APIConnectionError) don't accept response param + try: + raise raised_exception_obj( + message=error_str, + llm_provider=custom_llm_provider, + model=model, + response=response, + ) + except TypeError: + # Exception doesn't accept response parameter + raise raised_exception_obj( + message=error_str, + llm_provider=custom_llm_provider, + model=model, + ) def exception_type( # type: ignore # noqa: PLR0915 diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 49292545208..81ba717ab3f 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -1533,6 +1533,7 @@ class AWSEventStreamDecoder: ) ], id=self.response_id, + model=self.model, usage=usage, provider_specific_fields=model_response_provider_specific_fields, ) diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index 71c418757bf..5efd3ba1d9f 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -236,6 +236,7 @@ class BedrockPassthroughConfig( if len(all_translated_chunks) > 0: model_response = stream_chunk_builder( chunks=all_translated_chunks, + logging_obj=litellm_logging_obj, ) return model_response return None diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 9c8700daf83..8c98cc54050 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -190,46 +190,14 @@ class OllamaChatConfig(BaseConfig): else: optional_params["think"] = value in {"low", "medium", "high"} ### FUNCTION CALLING LOGIC ### + # Ollama 0.4+ supports native tool calling - pass tools directly + # and let Ollama handle model capability detection + # Fixes: https://github.com/BerriAI/litellm/issues/18922 if param == "tools": - ## CHECK IF MODEL SUPPORTS TOOL CALLING ## - try: - model_info = litellm.get_model_info( - model=model, custom_llm_provider="ollama" - ) - if model_info.get("supports_function_calling") is True: - optional_params["tools"] = value - else: - raise Exception - except Exception: - optional_params["format"] = "json" - litellm.add_function_to_prompt = ( - True # so that main.py adds the function call to the prompt - ) - optional_params["functions_unsupported_model"] = value - - if len(optional_params["functions_unsupported_model"]) == 1: - optional_params["function_name"] = optional_params[ - "functions_unsupported_model" - ][0]["function"]["name"] + optional_params["tools"] = value if param == "functions": - ## CHECK IF MODEL SUPPORTS TOOL CALLING ## - try: - model_info = litellm.get_model_info( - model=model, custom_llm_provider="ollama" - ) - if model_info.get("supports_function_calling") is True: - optional_params["tools"] = value - else: - raise Exception - except Exception: - optional_params["format"] = "json" - litellm.add_function_to_prompt = ( - True # so that main.py adds the function call to the prompt - ) - optional_params["functions_unsupported_model"] = ( - non_default_params.get("functions") - ) + optional_params["tools"] = value non_default_params.pop("tool_choice", None) # causes ollama requests to hang non_default_params.pop("functions", None) # causes ollama requests to hang return optional_params @@ -431,6 +399,10 @@ class OllamaChatConfig(BaseConfig): _message = litellm.Message(**response_json_message) model_response.choices[0].message = _message # type: ignore + # Set finish_reason to "tool_calls" when tool_calls are present + # Fixes: https://github.com/BerriAI/litellm/issues/18922 + if _message.tool_calls: + model_response.choices[0].finish_reason = "tool_calls" model_response.created = int(time.time()) model_response.model = "ollama_chat/" + model prompt_tokens = response_json.get("prompt_eval_count", litellm.token_counter(messages=messages)) # type: ignore @@ -563,6 +535,10 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): if chunk["done"] is True: finish_reason = chunk.get("done_reason", "stop") + # Override finish_reason when tool_calls are present + # Fixes: https://github.com/BerriAI/litellm/issues/18922 + if tool_calls is not None: + finish_reason = "tool_calls" choices = [ StreamingChoices( delta=delta, diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index e2cf9f46104..d0ed3f165cc 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -362,7 +362,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if has_stream_ended: # convert to model response model_response = cast( - ModelResponse, stream_chunk_builder(chunks=responses_so_far) + ModelResponse, stream_chunk_builder(chunks=responses_so_far, logging_obj=litellm_logging_obj) ) # run process_output_response await self.process_output_response( diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index c7e6a77b96f..5944705258e 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -22,7 +22,7 @@ from ...base_llm.audio_transcription.transformation import ( from ...openai.transcriptions.whisper_transformation import ( OpenAIWhisperAudioTranscriptionConfig, ) -from ..common_utils import IBMWatsonXMixin, _get_api_params +from ..common_utils import IBMWatsonXMixin class IBMWatsonXAudioTranscriptionConfig( @@ -48,7 +48,7 @@ class IBMWatsonXAudioTranscriptionConfig( ) -> Dict: """ Validate environment for audio transcription. - + Removes Content-Type header so httpx can set multipart/form-data automatically. """ result = IBMWatsonXMixin.validate_environment( @@ -88,31 +88,37 @@ class IBMWatsonXAudioTranscriptionConfig( ) -> AudioTranscriptionRequestData: """ Transform the audio transcription request for WatsonX. - + WatsonX expects multipart/form-data with: - file: the audio file - model: the model name (without watsonx/ prefix) - project_id: the project ID (as form field, not query param) + - space_id: the space ID (as form field, not query param) - other optional params """ # Use common utility to process the audio file processed_audio = process_audio_file(audio_file) - - # Get API params to extract project_id - api_params = _get_api_params(params=optional_params.copy()) - + project_id = optional_params.get("project_id") or optional_params.get( + "watsonx_project" + ) + space_id = optional_params.get("space_id") + # api_params = _get_api_params(params=optional_params, model=model) + # Initialize form data with required fields - form_data: WatsonXAudioTranscriptionRequestBody = { - "model": model, - "project_id": api_params.get("project_id", ""), - } - + form_data: WatsonXAudioTranscriptionRequestBody = {"model": model} + + # Only add project_id or space_id if they were explicitly provided by the user + if project_id: + form_data["project_id"] = project_id + elif space_id: + form_data["space_id"] = space_id + # Add supported OpenAI params to form data supported_params = self.get_supported_openai_params(model) for key, value in optional_params.items(): if key in supported_params and value is not None: form_data[key] = value # type: ignore - + # Prepare files dict with the audio file files = { "file": ( @@ -121,10 +127,10 @@ class IBMWatsonXAudioTranscriptionConfig( processed_audio.content_type, ) } - + # Convert TypedDict to regular dict for AudioTranscriptionRequestData form_data_dict: Dict[str, Any] = dict(form_data) - + return AudioTranscriptionRequestData(data=form_data_dict, files=files) def get_complete_url( @@ -140,8 +146,8 @@ class IBMWatsonXAudioTranscriptionConfig( Construct the complete URL for WatsonX audio transcription. URL format: {api_base}/ml/v1/audio/transcriptions?version={version} - - Note: project_id is sent as form data, not as a query parameter + + Note: project_id or space_id is sent as form data, not as a query parameter """ # Get base URL url = self._get_base_url(api_base=api_base) @@ -151,9 +157,10 @@ class IBMWatsonXAudioTranscriptionConfig( url = f"{url}/ml/v1/audio/transcriptions" # Add version parameter (only version in query string, not project_id) - api_version = optional_params.get( - "api_version", None - ) or litellm.WATSONX_DEFAULT_API_VERSION + api_version = ( + optional_params.get("api_version", None) + or litellm.WATSONX_DEFAULT_API_VERSION + ) url = f"{url}?version={api_version}" return url @@ -164,7 +171,7 @@ class IBMWatsonXAudioTranscriptionConfig( ) -> TranscriptionResponse: """ Transform the audio transcription response from WatsonX. - + WatsonX may include a 'model' field in the response, which needs to be removed before creating the TranscriptionResponse object. """ @@ -179,26 +186,30 @@ class IBMWatsonXAudioTranscriptionConfig( # TranscriptionResponse only accepts 'text' and 'usage' in __init__() text = raw_response_json.get("text") usage = raw_response_json.get("usage") - + # Create response with only valid fields response_kwargs = {} if text is not None: response_kwargs["text"] = text if usage is not None: response_kwargs["usage"] = usage - + if not response_kwargs: raise ValueError( "Invalid response format. Received response does not match the expected format. Got: ", raw_response_json, ) - + response = TranscriptionResponse(**response_kwargs) - + # Add other fields using dictionary-style assignment (like duration, task, etc.) # Skip fields that TranscriptionResponse doesn't accept in __init__() for key, value in raw_response_json.items(): - if key not in ["text", "usage", "model"]: # text/usage already set, model should be excluded + if key not in [ + "text", + "usage", + "model", + ]: # text/usage already set, model should be excluded response[key] = value - + return response diff --git a/litellm/llms/watsonx/chat/handler.py b/litellm/llms/watsonx/chat/handler.py index bc0effe4a1a..40ccc45497b 100644 --- a/litellm/llms/watsonx/chat/handler.py +++ b/litellm/llms/watsonx/chat/handler.py @@ -40,7 +40,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler): streaming_decoder: Optional[CustomStreamingDecoder] = None, fake_stream: bool = False, ): - api_params = _get_api_params(params=optional_params) + api_params = _get_api_params(params=optional_params, model=model) ## UPDATE HEADERS headers = watsonx_chat_transformation.validate_environment( diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index 0bb96673ef6..157493a4ce8 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -10,7 +10,6 @@ from litellm import verbose_logger from litellm.secret_managers.main import get_secret_str from litellm.types.llms.watsonx import ( WatsonXAIEndpoint, - WatsonXAPIParams, WatsonXModelPattern, ) @@ -115,18 +114,6 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): ) return url - def _prepare_payload(self, model: str, api_params: WatsonXAPIParams) -> dict: - """ - Prepare payload for deployment models. - Deployment models cannot have 'model_id' or 'model' in the request body. - """ - payload: dict = {} - payload["model_id"] = None if model.startswith("deployment/") else model - payload["project_id"] = ( - None if model.startswith("deployment/") else api_params["project_id"] - ) - return payload - @staticmethod def _apply_prompt_template_core( model: str, messages: List[Dict[str, str]], hf_template_fn diff --git a/litellm/llms/watsonx/common_utils.py b/litellm/llms/watsonx/common_utils.py index 0207020534c..774f6dc1f3d 100644 --- a/litellm/llms/watsonx/common_utils.py +++ b/litellm/llms/watsonx/common_utils.py @@ -80,9 +80,7 @@ def _generate_watsonx_token(api_key: Optional[str], token: Optional[str]) -> str return token -def _get_api_params( - params: dict, -) -> WatsonXAPIParams: +def _get_api_params(params: dict, model: Optional[str] = None) -> WatsonXAPIParams: """ Find watsonx.ai credentials in the params or environment variables and return the headers for authentication. """ @@ -118,10 +116,15 @@ def _get_api_params( or get_secret_str("SPACE_ID") ) - if project_id is None: + if ( + project_id is None + and space_id is None + and model is not None + and not model.startswith("deployment/") + ): raise WatsonXAIError( status_code=401, - message="Error: Watsonx project_id not set. Set WX_PROJECT_ID in environment variables or pass in as a parameter.", + message="Error: Watsonx project_id and space_id not set. Set WX_PROJECT_ID or WX_SPACE_ID in environment variables or pass in as a parameter.", ) return WatsonXAPIParams( @@ -146,7 +149,9 @@ async def _aconvert_watsonx_messages_core( model_prompt_dict = custom_prompt_dict[model] return ptf.custom_prompt( messages=messages, - role_dict=model_prompt_dict.get("role_dict", model_prompt_dict.get("roles")), + role_dict=model_prompt_dict.get( + "role_dict", model_prompt_dict.get("roles") + ), initial_prompt_value=model_prompt_dict.get("initial_prompt_value", ""), final_prompt_value=model_prompt_dict.get("final_prompt_value", ""), bos_token=model_prompt_dict.get("bos_token", ""), @@ -180,7 +185,9 @@ def _convert_watsonx_messages_core( model_prompt_dict = custom_prompt_dict[model] return ptf.custom_prompt( messages=messages, - role_dict=model_prompt_dict.get("role_dict", model_prompt_dict.get("roles")), + role_dict=model_prompt_dict.get( + "role_dict", model_prompt_dict.get("roles") + ), initial_prompt_value=model_prompt_dict.get("initial_prompt_value", ""), final_prompt_value=model_prompt_dict.get("final_prompt_value", ""), bos_token=model_prompt_dict.get("bos_token", ""), @@ -200,7 +207,10 @@ def _convert_watsonx_messages_core( async def aconvert_watsonx_messages_to_prompt( - model: str, messages: List[AllMessageValues], provider: str, custom_prompt_dict: Dict + model: str, + messages: List[AllMessageValues], + provider: str, + custom_prompt_dict: Dict, ) -> str: """Async version of convert_watsonx_messages_to_prompt""" from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig @@ -215,7 +225,10 @@ async def aconvert_watsonx_messages_to_prompt( def convert_watsonx_messages_to_prompt( - model: str, messages: List[AllMessageValues], provider: str, custom_prompt_dict: Dict + model: str, + messages: List[AllMessageValues], + provider: str, + custom_prompt_dict: Dict, ) -> str: """Sync version of convert_watsonx_messages_to_prompt""" from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig @@ -254,7 +267,8 @@ class IBMWatsonXMixin: ) zen_api_key = cast( Optional[str], - optional_params.pop("zen_api_key", None) or get_secret_str("WATSONX_ZENAPIKEY"), + optional_params.pop("zen_api_key", None) + or get_secret_str("WATSONX_ZENAPIKEY"), ) if token: headers["Authorization"] = f"Bearer {token}" @@ -360,5 +374,8 @@ class IBMWatsonXMixin: {} ) # Deployment models do not support 'space_id' or 'project_id' in their payload payload["model_id"] = model - payload["project_id"] = api_params["project_id"] + if api_params["project_id"] is not None: + payload["project_id"] = api_params["project_id"] + else: + payload["space_id"] = api_params["space_id"] return payload diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py index 3c1229ecd2b..7180e12162a 100644 --- a/litellm/llms/watsonx/completion/transformation.py +++ b/litellm/llms/watsonx/completion/transformation.py @@ -228,13 +228,17 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): "us-south", ] - def _build_request_payload(self, model: str, prompt: str, optional_params: Dict) -> Dict: + def _build_request_payload( + self, model: str, prompt: str, optional_params: Dict + ) -> Dict: """Shared logic to build request payload""" extra_body_params = optional_params.pop("extra_body", {}) optional_params.update(extra_body_params) - watsonx_api_params = _get_api_params(params=optional_params) - watsonx_auth_payload = self._prepare_payload(model=model, api_params=watsonx_api_params) - + watsonx_api_params = _get_api_params(params=optional_params, model=model) + watsonx_auth_payload = self._prepare_payload( + model=model, api_params=watsonx_api_params + ) + return { "input": prompt, "moderations": optional_params.pop("moderations", {}), @@ -242,21 +246,43 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): **watsonx_auth_payload, } - async def atransform_request(self, model: str, messages: List[AllMessageValues], optional_params: Dict, litellm_params: Dict, headers: Dict) -> Dict: + async def atransform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: Dict, + litellm_params: Dict, + headers: Dict, + ) -> Dict: """Async version of transform_request""" from litellm.llms.watsonx.common_utils import ( aconvert_watsonx_messages_to_prompt, ) - + provider = model.split("/")[0] - prompt = await aconvert_watsonx_messages_to_prompt(model=model, messages=messages, provider=provider, custom_prompt_dict={}) - return self._build_request_payload(model=model, prompt=prompt, optional_params=optional_params) - - def transform_request(self, model: str, messages: List[AllMessageValues], optional_params: Dict, litellm_params: Dict, headers: Dict) -> Dict: + prompt = await aconvert_watsonx_messages_to_prompt( + model=model, messages=messages, provider=provider, custom_prompt_dict={} + ) + return self._build_request_payload( + model=model, prompt=prompt, optional_params=optional_params + ) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: Dict, + litellm_params: Dict, + headers: Dict, + ) -> Dict: """Sync version of transform_request""" provider = model.split("/")[0] - prompt = convert_watsonx_messages_to_prompt(model=model, messages=messages, provider=provider, custom_prompt_dict={}) - return self._build_request_payload(model=model, prompt=prompt, optional_params=optional_params) + prompt = convert_watsonx_messages_to_prompt( + model=model, messages=messages, provider=provider, custom_prompt_dict={} + ) + return self._build_request_payload( + model=model, prompt=prompt, optional_params=optional_params + ) def transform_response( self, diff --git a/litellm/llms/watsonx/embed/transformation.py b/litellm/llms/watsonx/embed/transformation.py index 21f508da015..930212e3ef3 100644 --- a/litellm/llms/watsonx/embed/transformation.py +++ b/litellm/llms/watsonx/embed/transformation.py @@ -37,7 +37,7 @@ class IBMWatsonXEmbeddingConfig(IBMWatsonXMixin, BaseEmbeddingConfig): optional_params: dict, headers: dict, ) -> dict: - watsonx_api_params = _get_api_params(params=optional_params) + watsonx_api_params = _get_api_params(params=optional_params, model=model) watsonx_auth_payload = self._prepare_payload( model=model, api_params=watsonx_api_params, diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index fe53fe3b32b..c3da6892209 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -526,10 +526,24 @@ class InMemoryGuardrailHandler: ) default_on = litellm_params.default_on + + # Extract additional params from litellm_params to pass to custom guardrail + # This matches the behavior of other guardrail initializers (e.g., initialize_lakera) + # and aligns with the documented behavior for custom guardrails + if hasattr(litellm_params, "model_dump"): + extra_params = litellm_params.model_dump(exclude_none=True) + else: + extra_params = dict(litellm_params) if litellm_params else {} + + # Remove params that are handled explicitly or are internal + for key in ["guardrail", "mode", "default_on"]: + extra_params.pop(key, None) + _guardrail_callback = _guardrail_class( guardrail_name=guardrail["guardrail_name"], event_hook=mode, default_on=default_on, + **extra_params, ) litellm.logging_callback_manager.add_litellm_callback(_guardrail_callback) # type: ignore diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index ac39e5f4b2c..0c77b6f8510 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -851,7 +851,7 @@ async def delete_prompt( ) # Delete the prompt from the database - await prisma_client.db.litellm_prompttable.delete( + await prisma_client.db.litellm_prompttable.delete_many( where={"prompt_id": prompt_id} ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1f4174be616..a3254c32340 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -887,6 +887,10 @@ def get_openapi_schema(): from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema) + + # Fix Swagger UI execute path error when server_root_path is set + if server_root_path: + openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}] app.openapi_schema = openapi_schema return app.openapi_schema @@ -909,6 +913,10 @@ def custom_openapi(): from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema) + + # Fix Swagger UI execute path error when server_root_path is set + if server_root_path: + openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}] app.openapi_schema = openapi_schema return app.openapi_schema @@ -7086,8 +7094,8 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False) f"Provider token counting failed ({result.status_code}): {result.error_message}. " "Falling back to local tokenizer." ) - else: - # Success - return the result + elif result is not None: + # Success - return the result (only if not None) return result # Check if token counter is disabled before fallback diff --git a/litellm/router.py b/litellm/router.py index 2d49cc758e1..b77e3c9c299 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1229,6 +1229,7 @@ class Router: self, model: str, messages: List[Dict[str, str]], **kwargs ) -> Union[ModelResponse, CustomStreamWrapper]: model_name = None + deployment = None try: # pick the one that is available (lowest TPM/RPM) deployment = self.get_available_deployment( @@ -1291,6 +1292,9 @@ class Router: verbose_router_logger.info( f"litellm.completion(model={model_name})\033[31m Exception {str(e)}\033[0m" ) + # Set per-deployment num_retries on exception for retry logic + if deployment is not None: + self._set_deployment_num_retries_on_exception(e, deployment) raise e # fmt: off @@ -1510,7 +1514,7 @@ class Router: return FallbackStreamWrapper(stream_with_fallbacks()) - async def _acompletion( + async def _acompletion( # noqa: PLR0915 self, model: str, messages: List[Dict[str, str]], **kwargs ) -> Union[ModelResponse, CustomStreamWrapper,]: """ @@ -1520,6 +1524,7 @@ class Router: - in the semaphore, make a check against it's local rpm before running """ model_name = None + deployment = None _timeout_debug_deployment_dict = ( {} ) # this is a temporary dict to debug timeout issues @@ -1648,6 +1653,9 @@ class Router: "litellm_params", {} ).get("timeout", None) e.message += f"\n\nDeployment Info: request_timeout: {deployment_request_timeout_param}\ntimeout: {deployment_timeout_param}" + # Set per-deployment num_retries on exception for retry logic + if deployment is not None: + self._set_deployment_num_retries_on_exception(e, deployment) raise e except Exception as e: verbose_router_logger.info( @@ -1655,6 +1663,9 @@ class Router: ) if model_name is not None: self.fail_calls[model_name] += 1 + # Set per-deployment num_retries on exception for retry logic + if deployment is not None: + self._set_deployment_num_retries_on_exception(e, deployment) raise e def _update_kwargs_before_fallbacks( @@ -1678,6 +1689,24 @@ class Router: {"model_group": model, "model_group_alias": model_group_alias} ) + def _set_deployment_num_retries_on_exception( + self, exception: Exception, deployment: dict + ) -> None: + """ + Set num_retries from deployment litellm_params on the exception. + + This allows the retry logic in async_function_with_retries to use + per-deployment retry settings instead of the global setting. + """ + # Only set if exception doesn't already have num_retries + if hasattr(exception, "num_retries") and exception.num_retries is not None: # type: ignore + return + + litellm_params = deployment.get("litellm_params", {}) + dep_num_retries = litellm_params.get("num_retries") + if dep_num_retries is not None and isinstance(dep_num_retries, int): + exception.num_retries = dep_num_retries # type: ignore + def _update_kwargs_with_default_litellm_params( self, kwargs: dict, metadata_variable_name: Optional[str] = "metadata" ) -> None: diff --git a/litellm/types/llms/watsonx.py b/litellm/types/llms/watsonx.py index 6c42c3ecea0..137090b032e 100644 --- a/litellm/types/llms/watsonx.py +++ b/litellm/types/llms/watsonx.py @@ -5,7 +5,7 @@ from typing_extensions import NotRequired, TypedDict class WatsonXAPIParams(TypedDict): - project_id: str + project_id: Optional[str] space_id: Optional[str] region_name: Optional[str] @@ -19,7 +19,7 @@ class WatsonXCredentials(TypedDict): class WatsonXAudioTranscriptionRequestBody(TypedDict): """ WatsonX Audio Transcription API request body. - + Follows multipart/form-data format for WatsonX Whisper models. See: https://cloud.ibm.com/apidocs/watsonx-ai """ @@ -27,8 +27,11 @@ class WatsonXAudioTranscriptionRequestBody(TypedDict): model: str """Model name (e.g., 'whisper-large-v3-turbo')""" - project_id: str - """WatsonX project ID (required)""" + project_id: NotRequired[str] + """WatsonX project ID (optional)""" + + space_id: NotRequired[str] + """WatsonX space ID (optional)""" language: NotRequired[str] """Language code (e.g., 'en', 'es')""" @@ -64,6 +67,7 @@ class WatsonXAIEndpoint(str, Enum): class WatsonXModelPattern(str, Enum): """Model identifier patterns for WatsonX models""" + GRANITE_CHAT = "granite-chat" IBM_MISTRAL = "ibm-mistral" IBM_MISTRALAI = "ibm-mistralai" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1e685f88086..b5523385f08 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1250,6 +1250,14 @@ class Choices(OpenAIObject): params["message"] = message elif isinstance(message, dict): params["message"] = Message(**message) + elif isinstance(message, BaseModel): + # Normalize provider/OpenAI SDK message models into LiteLLM's Message type. + dump = ( + message.model_dump() + if hasattr(message, "model_dump") + else message.dict() + ) + params["message"] = Message(**dump) if logprobs is not None: if isinstance(logprobs, dict): params["logprobs"] = ChoiceLogprobs(**logprobs) @@ -1612,6 +1620,12 @@ class ModelResponseBase(OpenAIObject): _response_headers: Optional[dict] = None + def model_dump(self, **kwargs): + """Default to exclude_unset to avoid Pydantic serializer warnings for OpenAIObject-derived types.""" + if "exclude_unset" not in kwargs and "exclude_none" not in kwargs: + kwargs["exclude_unset"] = True + return super().model_dump(**kwargs) + class ModelResponseStream(ModelResponseBase): choices: List[StreamingChoices] @@ -1651,12 +1665,16 @@ class ModelResponseStream(ModelResponseBase): else: created = created - if ( - "usage" in kwargs - and kwargs["usage"] is not None - and isinstance(kwargs["usage"], dict) - ): - kwargs["usage"] = Usage(**kwargs["usage"]) + if "usage" in kwargs and kwargs["usage"] is not None: + if isinstance(kwargs["usage"], dict): + kwargs["usage"] = Usage(**kwargs["usage"]) + elif isinstance(kwargs["usage"], BaseModel): + dump = ( + kwargs["usage"].model_dump() + if hasattr(kwargs["usage"], "model_dump") + else kwargs["usage"].dict() + ) + kwargs["usage"] = Usage(**dump) kwargs["id"] = id kwargs["created"] = created @@ -1730,6 +1748,13 @@ class ModelResponse(ModelResponseBase): _new_choice = choice # type: ignore elif isinstance(choice, dict): _new_choice = Choices(**choice) # type: ignore + elif isinstance(choice, BaseModel): + dump = ( + choice.model_dump() + if hasattr(choice, "model_dump") + else choice.dict() + ) + _new_choice = Choices(**dump) # type: ignore else: _new_choice = choice new_choices.append(_new_choice) @@ -1748,6 +1773,11 @@ class ModelResponse(ModelResponseBase): if usage is not None: if isinstance(usage, dict): usage = Usage(**usage) + elif isinstance(usage, BaseModel): + dump = ( + usage.model_dump() if hasattr(usage, "model_dump") else usage.dict() + ) + usage = Usage(**dump) else: usage = usage elif stream is None or stream is False: @@ -3032,7 +3062,6 @@ class LlmProviders(str, Enum): XIAOMI_MIMO = "xiaomi_mimo" - # Create a set of all provider values for quick lookup LlmProvidersSet = {provider.value for provider in LlmProviders} diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 29c2d26981c..7c0db41d13a 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -3350,7 +3350,8 @@ async def test_bedrock_converse__streaming_passthrough(monkeypatch): mock_callback.assert_called_once() print(mock_callback.call_args.kwargs.keys()) assert "response_cost" in mock_callback.call_args.kwargs["kwargs"] - assert mock_callback.call_args.kwargs["kwargs"]["response_cost"] > 0 + response_cost = mock_callback.call_args.kwargs["kwargs"]["response_cost"] + assert response_cost is not None and response_cost > 0 assert "standard_logging_object" in mock_callback.call_args.kwargs["kwargs"] diff --git a/tests/llm_translation/test_watsonx.py b/tests/llm_translation/test_watsonx.py index e0c125a9017..d6a82b44969 100644 --- a/tests/llm_translation/test_watsonx.py +++ b/tests/llm_translation/test_watsonx.py @@ -1,17 +1,14 @@ import json import os import sys -from datetime import datetime -from unittest.mock import AsyncMock sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path import litellm from litellm import completion, embedding -from litellm.llms.watsonx.common_utils import IBMWatsonXMixin -from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler -from unittest.mock import patch, MagicMock, AsyncMock, Mock +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from unittest.mock import patch, Mock import pytest from typing import Optional @@ -188,6 +185,27 @@ def test_watsonx_chat_completions_endpoint(watsonx_chat_completion_call): assert "deployment" not in mock_post.call_args.kwargs["url"] +def test_watsonx_chat_completions_endpoint_space_id( + monkeypatch, watsonx_chat_completion_call +): + my_fake_space_id = "xxx-xxx-xxx-xxx-xxx" + monkeypatch.setenv("WATSONX_SPACE_ID", my_fake_space_id) + + monkeypatch.delenv("WATSONX_PROJECT_ID", raising=False) + + model = "watsonx/another-model" + messages = [{"role": "user", "content": "Test message"}] + + mock_post, _ = watsonx_chat_completion_call(model=model, messages=messages) + + assert mock_post.call_count == 1 + assert "deployment" not in mock_post.call_args.kwargs["url"] + + json_data = json.loads(mock_post.call_args.kwargs["data"]) + assert my_fake_space_id == json_data["space_id"] + assert not json_data.get("project_id") + + @pytest.mark.parametrize( "model", [ @@ -209,6 +227,27 @@ def test_watsonx_deployment_space_id(monkeypatch, watsonx_chat_completion_call, assert my_fake_space_id not in json_data +@pytest.mark.parametrize( + "model", + [ + "watsonx/deployment/", + "watsonx_text/deployment/", + ], +) +def test_watsonx_deployment(watsonx_chat_completion_call, model): + messages = [{"content": "Hello, how are you?", "role": "user"}] + mock_post, _ = watsonx_chat_completion_call( + model=model, + messages=messages, + ) + + assert mock_post.call_count == 1 + json_data = json.loads(mock_post.call_args.kwargs["data"]) + + # nor space_id or project_id is required by wx.ai API when inferencing deployment + assert "project_id" not in json_data and "space_id" not in json_data + + def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_call): my_fake_space_id = "xxx-xxx-xxx-xxx-xxx" monkeypatch.setenv("WATSONX_SPACE_ID", my_fake_space_id) @@ -217,4 +256,6 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca assert mock_post.call_count == 1 json_data = json.loads(mock_post.call_args.kwargs["data"]) - assert my_fake_space_id not in json_data + + # nor space_id or project_id is required by wx.ai API when inferencing deployment + assert "project_id" not in json_data and "space_id" not in json_data diff --git a/tests/logging_callback_tests/test_langsmith_unit_test.py b/tests/logging_callback_tests/test_langsmith_unit_test.py index bde2b944579..17b854b52f2 100644 --- a/tests/logging_callback_tests/test_langsmith_unit_test.py +++ b/tests/logging_callback_tests/test_langsmith_unit_test.py @@ -392,7 +392,6 @@ async def test_langsmith_key_based_logging(): "role": "assistant", "tool_calls": None, "function_call": None, - "provider_specific_fields": None, }, } ], diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index 9e742a83c6a..c7ad18cfb0c 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -12,6 +12,7 @@ sys.path.insert( from litellm.litellm_core_utils.exception_mapping_utils import ( ExceptionCheckers, exception_type, + extract_and_raise_litellm_exception, ) # Test cases for is_error_str_context_window_exceeded @@ -269,3 +270,79 @@ def test_vertex_ai_rate_limit_error_mapping(error_message, should_raise_rate_lim original_exception=original_exception, custom_llm_provider=custom_llm_provider, ) + + +class TestExtractAndRaiseLitellmException: + """Tests for extract_and_raise_litellm_exception function""" + + def test_extract_and_raise_api_connection_error_without_response(self): + """ + Test that APIConnectionError can be raised without response parameter. + + This is a regression test for the bug where extract_and_raise_litellm_exception + would fail with TypeError when trying to raise APIConnectionError with a + response parameter, since APIConnectionError doesn't accept that parameter. + + Relevant Issue: https://github.com/BerriAI/litellm/issues/XXXXX + """ + error_str = "litellm.APIConnectionError: GeminiException - some error message" + + with pytest.raises(litellm.APIConnectionError) as excinfo: + extract_and_raise_litellm_exception( + response=None, + error_str=error_str, + model="gemini/gemini-3-pro-preview", + custom_llm_provider="gemini", + ) + + assert "APIConnectionError" in str(excinfo.value) + + def test_extract_and_raise_bad_request_error_with_response(self): + """ + Test that BadRequestError can be raised with response parameter. + + BadRequestError does accept the response parameter, so this should work. + """ + error_str = "litellm.BadRequestError: Invalid request format" + + with pytest.raises(litellm.BadRequestError) as excinfo: + extract_and_raise_litellm_exception( + response=None, + error_str=error_str, + model="gpt-4", + custom_llm_provider="openai", + ) + + assert "BadRequestError" in str(excinfo.value) + + def test_extract_and_raise_context_window_exceeded_error(self): + """ + Test that ContextWindowExceededError can be raised. + """ + error_str = "litellm.ContextWindowExceededError: Token limit exceeded" + + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + extract_and_raise_litellm_exception( + response=None, + error_str=error_str, + model="gpt-4", + custom_llm_provider="openai", + ) + + assert "ContextWindowExceededError" in str(excinfo.value) + + def test_no_exception_raised_for_non_litellm_error(self): + """ + Test that no exception is raised for non-litellm error strings. + """ + error_str = "Some generic error that is not a litellm exception" + + # Should not raise any exception + result = extract_and_raise_litellm_exception( + response=None, + error_str=error_str, + model="gpt-4", + custom_llm_provider="openai", + ) + + assert result is None diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index fc4a3e43573..af6481a6cb0 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -323,3 +323,153 @@ class TestOllamaChatConfigResponseFormat: # and the code checks "if images is not None", an empty list will still be set assert "images" in result["messages"][0] assert result["messages"][0]["images"] == [] + + +class TestOllamaToolCalling: + """Tests for Ollama tool calling fixes. + + Issue: https://github.com/BerriAI/litellm/issues/18922 + """ + + def test_tools_passed_directly_without_capability_check(self): + """Test that tools are passed directly to Ollama without model capability checks. + + Previously, the code called litellm.get_model_info() which could fail + when Ollama runs on a remote server, causing a broken fallback. + Now tools are passed directly - Ollama 0.4+ handles capability detection. + """ + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + + optional_params = get_optional_params( + model="ollama_chat/qwen3:14b", + tools=tools, + custom_llm_provider="ollama_chat", + ) + + # Tools should be passed through directly + assert "tools" in optional_params + assert optional_params["tools"] == tools + # Should NOT trigger the broken fallback + assert "functions_unsupported_model" not in optional_params + assert "format" not in optional_params or optional_params.get("format") != "json" + + def test_finish_reason_tool_calls_non_streaming(self): + """Test that finish_reason is set to 'tool_calls' when tool_calls present. + + Previously, finish_reason was hardcoded to 'stop' even when tool_calls + were in the response, causing clients to ignore the tool calls. + """ + import json + from unittest.mock import MagicMock + + import litellm + from litellm.types.utils import Choices, Message, ModelResponse + + config = OllamaChatConfig() + + # Simulated Ollama response with tool_calls + ollama_response = { + "model": "qwen3:14b", + "created_at": "2025-01-11T00:00:00.000000Z", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "function": { + "name": "get_weather", + "arguments": {"location": "Tokyo"}, + } + } + ], + }, + "done": True, + "prompt_eval_count": 100, + "eval_count": 50, + } + + mock_response = MagicMock() + mock_response.json.return_value = ollama_response + mock_response.text = json.dumps(ollama_response) + + mock_logging = MagicMock() + + model_response = ModelResponse() + model_response.choices = [Choices(message=Message(content=""), index=0)] + + result = config.transform_response( + model="qwen3:14b", + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging, + request_data={}, + messages=[{"role": "user", "content": "Weather?"}], + optional_params={}, + litellm_params={}, + encoding=None, + api_key=None, + json_mode=False, + ) + + # finish_reason should be "tool_calls", not "stop" + assert result.choices[0].finish_reason == "tool_calls" + assert result.choices[0].message.tool_calls is not None + + def test_finish_reason_stop_when_no_tool_calls(self): + """Test that finish_reason remains 'stop' when no tool_calls present.""" + import json + from unittest.mock import MagicMock + + import litellm + from litellm.types.utils import Choices, Message, ModelResponse + + config = OllamaChatConfig() + + # Simulated Ollama response without tool_calls + ollama_response = { + "model": "qwen3:14b", + "created_at": "2025-01-11T00:00:00.000000Z", + "message": { + "role": "assistant", + "content": "Hello! How can I help you?", + }, + "done": True, + "prompt_eval_count": 100, + "eval_count": 50, + } + + mock_response = MagicMock() + mock_response.json.return_value = ollama_response + mock_response.text = json.dumps(ollama_response) + + mock_logging = MagicMock() + + model_response = ModelResponse() + model_response.choices = [Choices(message=Message(content=""), index=0)] + + result = config.transform_response( + model="qwen3:14b", + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging, + request_data={}, + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + encoding=None, + api_key=None, + json_mode=False, + ) + + # finish_reason should be "stop" (default behavior) + assert result.choices[0].finish_reason == "stop" + assert result.choices[0].message.tool_calls is None diff --git a/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py b/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py index fd5f8f3eff8..6ff53287e9d 100644 --- a/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py @@ -35,7 +35,7 @@ class TestWatsonXAudioTranscription: captured_request["headers"] = kwargs.get("headers", {}) captured_request["data"] = kwargs.get("data", {}) captured_request["files"] = kwargs.get("files", {}) - + mock_response = MagicMock() mock_response.json.return_value = { "text": "test transcription", @@ -44,7 +44,10 @@ class TestWatsonXAudioTranscription: mock_response.status_code = 200 return mock_response - with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=mock_post): + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=mock_post, + ): try: await litellm.atranscription( model="watsonx/whisper-large-v3-turbo", @@ -65,14 +68,16 @@ class TestWatsonXAudioTranscription: # Validate headers contain WatsonX auth assert "Authorization" in captured_request["headers"] - assert "Bearer test-bearer-token" in captured_request["headers"]["Authorization"] - + assert ( + "Bearer test-bearer-token" in captured_request["headers"]["Authorization"] + ) + # Validate Content-Type is NOT set (httpx sets multipart/form-data automatically) assert "Content-Type" not in captured_request["headers"] - + # Validate project_id is in form data, not URL assert captured_request["data"].get("project_id") == "test-project-123" - + # Validate file is in files dict assert "file" in captured_request["files"] @@ -80,7 +85,7 @@ class TestWatsonXAudioTranscription: async def test_watsonx_transcription_request_body(self): """ Test that litellm.transcription sends correct request body for WatsonX. - + Validates that: - Request uses multipart/form-data (data + files) - Model name has watsonx/ prefix removed @@ -93,7 +98,7 @@ class TestWatsonXAudioTranscription: async def mock_post(*args, **kwargs): captured_request["data"] = kwargs.get("data", {}) captured_request["files"] = kwargs.get("files", {}) - + mock_response = MagicMock() mock_response.json.return_value = { "text": "test transcription", @@ -102,7 +107,10 @@ class TestWatsonXAudioTranscription: mock_response.status_code = 200 return mock_response - with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=mock_post): + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=mock_post, + ): try: await litellm.atranscription( model="watsonx/whisper-large-v3-turbo", @@ -122,29 +130,31 @@ class TestWatsonXAudioTranscription: print("JSON DUMPS captured_request:") print(json.dumps(captured_request, indent=4, default=str)) - + # Model name should NOT have watsonx/ prefix assert data.get("model") == "whisper-large-v3-turbo" - + # project_id should be in form data assert data.get("project_id") == "test-project-123" - + # OpenAI params should be in form data assert data.get("language") == "en" assert data.get("temperature") == 0.5 # response_format should NOT be set by default - only send what user specifies assert "response_format" not in data - + # Validate file is in files dict (multipart/form-data) files = captured_request.get("files", {}) assert "file" in files - assert isinstance(files["file"], tuple) # Should be (filename, content, content_type) + assert isinstance( + files["file"], tuple + ) # Should be (filename, content, content_type) @pytest.mark.asyncio - async def test_watsonx_transcription_only_user_params_sent(self): + async def test_watsonx_transcription_only_user_params_sent_with_project_id(self): """ Test that only user-specified params are sent in request body to WatsonX. - + LiteLLM should NOT add extra params like response_format if user didn't specify them. """ captured_request = {} @@ -152,7 +162,7 @@ class TestWatsonXAudioTranscription: async def mock_post(*args, **kwargs): captured_request["data"] = kwargs.get("data", {}) captured_request["files"] = kwargs.get("files", {}) - + mock_response = MagicMock() mock_response.json.return_value = { "text": "test transcription", @@ -161,7 +171,10 @@ class TestWatsonXAudioTranscription: mock_response.status_code = 200 return mock_response - with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=mock_post): + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=mock_post, + ): try: # Minimal request - only required params await litellm.atranscription( @@ -176,20 +189,81 @@ class TestWatsonXAudioTranscription: pass # We just want to capture the request data = captured_request.get("data", {}) - + # These are the ONLY keys that should be in data expected_keys = {"model", "project_id"} actual_keys = set(data.keys()) - + assert actual_keys == expected_keys, ( f"Request body should only contain {expected_keys}, " f"but got {actual_keys}. " f"Extra keys: {actual_keys - expected_keys}" ) - + # Specifically verify response_format is NOT added - assert "response_format" not in data, "response_format should NOT be added by default" - + assert ( + "response_format" not in data + ), "response_format should NOT be added by default" + + # Verify file is sent separately + files = captured_request.get("files", {}) + assert "file" in files + + @pytest.mark.asyncio + async def test_watsonx_transcription_only_user_params_sent_with_space_id(self): + """ + Test that only user-specified params are sent in request body to WatsonX. + + LiteLLM should NOT add extra params like response_format if user didn't specify them. + """ + captured_request = {} + + async def mock_post(*args, **kwargs): + captured_request["data"] = kwargs.get("data", {}) + captured_request["files"] = kwargs.get("files", {}) + + mock_response = MagicMock() + mock_response.json.return_value = { + "text": "test transcription", + "duration": 1.0, + } + mock_response.status_code = 200 + return mock_response + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=mock_post, + ): + try: + # Minimal request - only required params + await litellm.atranscription( + model="watsonx/whisper-large-v3-turbo", + file=b"fake_audio_data", + api_base="https://us-south.ml.cloud.ibm.com", + api_key="test-api-key", + space_id="test-space_id-123", + token="test-bearer-token", + ) + except Exception: + pass # We just want to capture the request + + data = captured_request.get("data", {}) + + # These are the ONLY keys that should be in data + expected_keys = {"model", "space_id"} + actual_keys = set(data.keys()) + + assert actual_keys == expected_keys, ( + f"Request body should only contain {expected_keys}, " + f"but got {actual_keys}. " + f"Extra keys: {actual_keys - expected_keys}" + ) + + # Specifically verify response_format is NOT added + assert ( + "response_format" not in data + ), "response_format should NOT be added by default" + # Verify file is sent separately files = captured_request.get("files", {}) assert "file" in files @@ -198,7 +272,7 @@ class TestWatsonXAudioTranscription: """ Test that transform_audio_transcription_response removes the 'model' field from WatsonX response before creating TranscriptionResponse. - + This test ensures that when WatsonX returns a response with a 'model' field, it is removed before creating the TranscriptionResponse object, since TranscriptionResponse doesn't accept a 'model' parameter. @@ -219,13 +293,13 @@ class TestWatsonXAudioTranscription: # Verify the result is a TranscriptionResponse assert isinstance(result, TranscriptionResponse) - + # Verify the text is correct assert result.text == "Hello, this is a test transcription." - + # Verify duration is set via dictionary assignment assert result["duration"] == 5.5 - + # Verify the model field is NOT in the serialized result # Check via model_dump() or dict() to ensure it's not in the output try: @@ -233,7 +307,7 @@ class TestWatsonXAudioTranscription: except AttributeError: # Fallback for pydantic v1 result_dict = result.dict() - + # The 'model' field should not be in the result assert "model" not in result_dict, "Model field should be removed from response" @@ -250,15 +324,17 @@ class TestWatsonXAudioTranscription: "text": "Hello, this is a test transcription.", "duration": 5.5, } - mock_response.text = '{"text": "Hello, this is a test transcription.", "duration": 5.5}' + mock_response.text = ( + '{"text": "Hello, this is a test transcription.", "duration": 5.5}' + ) result = handler.transform_audio_transcription_response(mock_response) # Verify the result is a TranscriptionResponse assert isinstance(result, TranscriptionResponse) - + # Verify the text is correct assert result.text == "Hello, this is a test transcription." - + # Verify duration is set via dictionary assignment assert result["duration"] == 5.5 diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 57ec019e420..33e000143c8 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -201,6 +201,10 @@ ignored_keys = [ "metadata.usage_object", "metadata.cold_storage_object_key", "metadata.additional_usage_values.prompt_tokens_details.cache_creation_tokens", + "metadata.additional_usage_values.completion_tokens_details", + "metadata.additional_usage_values.prompt_tokens_details", + "metadata.additional_usage_values.cache_creation_input_tokens", + "metadata.additional_usage_values.cache_read_input_tokens", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", ] diff --git a/tests/test_litellm/proxy/test_swagger_chat_completions.py b/tests/test_litellm/proxy/test_swagger_chat_completions.py index b973eab6213..968443ef4d7 100644 --- a/tests/test_litellm/proxy/test_swagger_chat_completions.py +++ b/tests/test_litellm/proxy/test_swagger_chat_completions.py @@ -307,4 +307,43 @@ class TestSwaggerChatCompletions: # Verify required fields are present in test request required_fields = schema_def.get("required", []) for required_field in required_fields: - assert required_field in test_request, f"Required field '{required_field}' should be in test request" \ No newline at end of file + assert required_field in test_request, f"Required field '{required_field}' should be in test request" + + def test_openapi_schema_servers_url_with_root_path(self): + """ + Test that OpenAPI schema includes correct servers URL when server_root_path is set. + This ensures Swagger UI works correctly with reverse proxies and subpath deployments. + """ + from unittest.mock import patch + from litellm.proxy.proxy_server import get_openapi_schema, custom_openapi, app + + # Test cases: (server_root_path, expected_servers_url) + # Note: empty string is falsy in Python, so servers won't be set + test_cases = [ + ("/litellm", "/litellm"), + ("/litellm/", "/litellm"), # trailing slash should be removed + ("litellm", "/litellm"), # missing leading slash should be added + ("/api/v1", "/api/v1"), + ] + + for root_path, expected_url in test_cases: + # Clear cached schema + app.openapi_schema = None + + with patch("litellm.proxy.proxy_server.server_root_path", root_path): + # Test get_openapi_schema + schema = get_openapi_schema() + + # Should have servers field with correct URL + assert "servers" in schema, f"servers field should exist when server_root_path={root_path}" + assert schema["servers"][0]["url"] == expected_url, \ + f"Expected servers URL '{expected_url}', got '{schema['servers'][0]['url']}' for root_path '{root_path}'" + + # Test custom_openapi as well + app.openapi_schema = None + with patch("litellm.proxy.proxy_server.server_root_path", root_path): + schema = custom_openapi() + + assert "servers" in schema, f"servers field should exist in custom_openapi when server_root_path={root_path}" + assert schema["servers"][0]["url"] == expected_url, \ + f"Expected servers URL '{expected_url}' in custom_openapi, got '{schema['servers'][0]['url']}'" \ No newline at end of file diff --git a/tests/test_litellm/test_model_response_normalization.py b/tests/test_litellm/test_model_response_normalization.py new file mode 100644 index 00000000000..57281d3c1fc --- /dev/null +++ b/tests/test_litellm/test_model_response_normalization.py @@ -0,0 +1,61 @@ +import warnings + +import pytest + +from litellm.types.utils import Choices, Message, ModelResponse + + +def test_modelresponse_normalizes_openai_base_models() -> None: + # OpenAI SDK returns Pydantic BaseModel objects for message/choice. + # LiteLLM should normalize these into its own internal `Message` / `Choices` types. + from openai.types.chat.chat_completion import Choice as OpenAIChoice + from openai.types.chat.chat_completion_message import ChatCompletionMessage + + message = ChatCompletionMessage(role="assistant", content="hi") + choice = OpenAIChoice(finish_reason="stop", index=0, message=message, logprobs=None) + + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always") + response = ModelResponse(model="gpt-4o-mini", choices=[choice]) + _ = response.model_dump() + + assert isinstance(response.choices[0], Choices) + assert isinstance(response.choices[0].message, Message) + + assert not any( + "Pydantic serializer warnings" in str(w.message) + for w in captured + if isinstance(w.message, Warning) + ) + + +def test_modelresponse_serialization_avoids_pydantic_warnings() -> None: + pytest.importorskip("openai") + from openai.types.chat import ChatCompletion as OpenAIChatCompletion + + openai_completion = OpenAIChatCompletion( + id="test-1", + created=1719868600, + model="gpt-4o-mini", + object="chat.completion", + choices=[ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "hi"}, + "logprobs": None, + } + ], + usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + ) + + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always") + response = ModelResponse(**openai_completion.model_dump()) + _ = response.model_dump(exclude_none=True) + + assert not any( + "PydanticSerializationUnexpectedValue" in str(w.message) + or "Pydantic serializer warnings" in str(w.message) + for w in captured + ) diff --git a/tests/test_litellm/test_per_deployment_num_retries.py b/tests/test_litellm/test_per_deployment_num_retries.py new file mode 100644 index 00000000000..4021ca28073 --- /dev/null +++ b/tests/test_litellm/test_per_deployment_num_retries.py @@ -0,0 +1,157 @@ +""" +Unit tests for per-deployment num_retries in litellm_params +GitHub Issue: #18968 - Per-deployment max_retries/num_retries in litellm_params is not used in retry logic +""" + +import pytest +from unittest.mock import MagicMock, patch + +from litellm import Router + + +class TestPerDeploymentNumRetries: + """Test that per-deployment num_retries in litellm_params is correctly used.""" + + def test_set_deployment_num_retries_on_exception(self): + """ + Test that _set_deployment_num_retries_on_exception sets num_retries + on the exception from the deployment's litellm_params. + """ + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "test-key", + "num_retries": 5, # Per-deployment setting + }, + }, + ], + num_retries=1, # Global setting + ) + + deployment = router.model_list[0] + + # Create a mock exception without num_retries + class MockException(Exception): + pass + + exc = MockException("test error") + assert not hasattr(exc, "num_retries") or exc.num_retries is None + + # Call the helper + router._set_deployment_num_retries_on_exception(exc, deployment) + + # Verify num_retries was set from deployment + assert exc.num_retries == 5 + + def test_set_deployment_num_retries_does_not_override_existing(self): + """ + Test that _set_deployment_num_retries_on_exception does NOT override + if exception already has num_retries set. + """ + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "test-key", + "num_retries": 5, + }, + }, + ], + num_retries=1, + ) + + deployment = router.model_list[0] + + # Create an exception that already has num_retries + class MockException(Exception): + num_retries = 10 # Already set + + exc = MockException("test error") + + # Call the helper + router._set_deployment_num_retries_on_exception(exc, deployment) + + # Verify num_retries was NOT overridden + assert exc.num_retries == 10 + + def test_deployment_without_num_retries(self): + """ + Test that _set_deployment_num_retries_on_exception does nothing + if deployment has no num_retries set. + """ + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "test-key", + # No num_retries set + }, + }, + ], + num_retries=3, + ) + + deployment = router.model_list[0] + + class MockException(Exception): + pass + + exc = MockException("test error") + + # Call the helper + router._set_deployment_num_retries_on_exception(exc, deployment) + + # Verify num_retries was not set (deployment has no num_retries) + assert not hasattr(exc, "num_retries") or exc.num_retries is None + + def test_request_level_num_retries_takes_precedence(self): + """ + Test that request-level num_retries (passed in kwargs) is still respected. + """ + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "test-key", + "num_retries": 5, + }, + }, + ], + num_retries=1, + ) + + # Pass num_retries in request kwargs - this should take precedence + kwargs = {"num_retries": 10} + router._update_kwargs_before_fallbacks(model="test-model", kwargs=kwargs) + assert kwargs["num_retries"] == 10 # Request-level takes precedence + + def test_global_num_retries_used_when_no_deployment_setting(self): + """ + Test that global num_retries is used when deployment has no num_retries. + """ + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "test-key", + # No num_retries set + }, + }, + ], + num_retries=7, # Global setting + ) + + kwargs = {} + router._update_kwargs_before_fallbacks(model="test-model", kwargs=kwargs) + assert kwargs["num_retries"] == 7 # Uses global diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index ad023cf6b1f..6279e96305f 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1200,9 +1200,7 @@ async def test_acompletion_streaming_disable_fallbacks_midstream(): print("\n=== Test 1: disable_fallbacks=True with original_exception ===") # Create an original exception to wrap - from litellm.llms.anthropic.chat.anthropic_chat_transformation import ( - AnthropicError, - ) + from litellm.llms.anthropic.common_utils import AnthropicError original_error = AnthropicError( status_code=500, diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index f747b5b2599..70644f12d20 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -54,7 +54,7 @@ import { makeOpenAIResponsesRequest } from "../llm_calls/responses_api"; import CodeInterpreterOutput from "./CodeInterpreterOutput"; import { useCodeInterpreter } from "./useCodeInterpreter"; import { Agent, fetchAvailableAgents } from "../llm_calls/fetch_agents"; -import { makeA2AStreamMessageRequest } from "../llm_calls/a2a_send_message"; +import { makeA2AStreamMessageRequest, makeA2ASendMessageRequest } from "../llm_calls/a2a_send_message"; import A2AMetrics from "./A2AMetrics"; import { A2ATaskMetadata } from "./types"; import MCPEventsDisplay, { MCPEvent } from "./MCPEventsDisplay"; @@ -1037,7 +1037,7 @@ const ChatUI: React.FC = ({ // Handle A2A agent calls (separate from model-based calls) - use streaming if (endpointType === EndpointType.A2A_AGENTS && selectedAgent) { - await makeA2AStreamMessageRequest( + await makeA2ASendMessageRequest( selectedAgent, inputMessage, (chunk, model) => updateTextUI("assistant", chunk, model), diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000000..bda0207302b --- /dev/null +++ b/uv.lock @@ -0,0 +1,3 @@ +version = 1 +revision = 3 +requires-python = ">=3.13"