From 498b8d85cde655cc159bc11f360bd30c90363b59 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 25 Jun 2026 22:16:06 -0700 Subject: [PATCH] fix(guardrails/anthropic): apply compressed structured_messages on /v1/messages and fix guardrail_information logging Two bugs fixed for /v1/messages (Anthropic native endpoint) with headroom prompt compression guardrail: 1. guardrail_information null in spend logs: function_setup stored a .copy() of litellm_metadata as litellm_params["metadata"]. Guardrails write to litellm_metadata after this point; the copy didn't reflect those mutations. Changed to use a reference so guardrail writes are visible at logging time. 2. Compressed structured_messages not applied: AnthropicMessagesHandler.process_input_messages sent OpenAI-format structured_messages to the guardrail but never read back the compressed result. Added identity-check (is not) to detect when guardrail returns different structured_messages and convert them back to Anthropic format via anthropic_messages_pt before sending the request. Uses identity check (not equality) to distinguish pass-through guardrails (which echo the original structured_messages unchanged) from compression guardrails (which return a new list object with fewer messages). --- .../chat/guardrail_translation/handler.py | 103 +- litellm/utils.py | 1939 ++++------------- .../test_anthropic_guardrail_handler.py | 142 +- 3 files changed, 640 insertions(+), 1544 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 7d5944b4cc2..7e8fd6af422 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -125,9 +125,7 @@ class AnthropicMessagesHandler(BaseTranslation): texts_to_check: List[str] = [] images_to_check: List[str] = [] - tools_to_check: List[ChatCompletionToolParam] = ( - chat_completion_compatible_request.get("tools", []) - ) + tools_to_check: List[ChatCompletionToolParam] = chat_completion_compatible_request.get("tools", []) task_mappings: List[Tuple[int, Optional[int]]] = [] # Step 1: Extract all text content and images @@ -164,6 +162,8 @@ class AnthropicMessagesHandler(BaseTranslation): guardrailed_texts = guardrailed_inputs.get("texts", []) guardrailed_tools = guardrailed_inputs.get("tools") + guardrailed_structured_messages = guardrailed_inputs.get("structured_messages") + if guardrailed_tools is not None: # Convert tools back from OpenAI format to Anthropic format anthropic_config = AnthropicConfig() @@ -172,19 +172,35 @@ class AnthropicMessagesHandler(BaseTranslation): converted_tool, mcp_server = anthropic_config._map_tool_helper(tool) if converted_tool is not None: anthropic_tools.append(converted_tool) - # Note: MCP servers are handled separately in the main transformation data["tools"] = anthropic_tools - # Step 3: Map guardrail responses back to original message structure - await self._apply_guardrail_responses_to_input( - messages=messages, - responses=guardrailed_texts, - task_mappings=task_mappings, - ) + # If guardrail returned modified structured_messages (OpenAI format), convert + # them back to Anthropic format and replace the request messages wholesale. + # Only do this when the returned messages differ from what we sent — a + # pass-through guardrail echoes the original structured_messages unchanged. + if ( + guardrailed_structured_messages is not None + and guardrailed_structured_messages is not structured_messages + ): + from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_messages_pt, + ) - verbose_proxy_logger.debug( - "Anthropic Messages: Processed input messages: %s", messages - ) + model = data.get("model", "") + data["messages"] = anthropic_messages_pt( + messages=guardrailed_structured_messages, + model=model, + llm_provider="anthropic", + ) + else: + # Step 3: Map guardrail responses back to original message structure + await self._apply_guardrail_responses_to_input( + messages=messages, + responses=guardrailed_texts, + task_mappings=task_mappings, + ) + + verbose_proxy_logger.debug("Anthropic Messages: Processed input messages: %s", messages) return data @@ -288,9 +304,7 @@ class AnthropicMessagesHandler(BaseTranslation): elif isinstance(content, list) and content_idx_optional is not None: # Replace specific text item in list content - messages[msg_idx]["content"][content_idx_optional]["text"] = ( - guardrail_response - ) + messages[msg_idx]["content"][content_idx_optional]["text"] = guardrail_response async def process_output_response( self, @@ -369,9 +383,7 @@ class AnthropicMessagesHandler(BaseTranslation): task_mappings=task_mappings, ) - verbose_proxy_logger.debug( - "Anthropic Messages: Processed output response: %s", response - ) + verbose_proxy_logger.debug("Anthropic Messages: Processed output response: %s", response) return response @@ -391,20 +403,14 @@ class AnthropicMessagesHandler(BaseTranslation): has_ended = self._check_streaming_has_ended(responses_so_far) if has_ended: # build the model response from the responses_so_far - built_response = ( - AnthropicPassthroughLoggingHandler._build_complete_streaming_response( - all_chunks=responses_so_far, - litellm_logging_obj=cast("LiteLLMLoggingObj", litellm_logging_obj), - model="", - ) + built_response = AnthropicPassthroughLoggingHandler._build_complete_streaming_response( + all_chunks=responses_so_far, + litellm_logging_obj=cast("LiteLLMLoggingObj", litellm_logging_obj), + model="", ) # Check if model_response is valid and has choices before accessing - if ( - built_response is not None - and hasattr(built_response, "choices") - and built_response.choices - ): + if built_response is not None and hasattr(built_response, "choices") and built_response.choices: model_response = cast(ModelResponse, built_response) first_choice = cast(Choices, model_response.choices[0]) tool_calls_list = cast( @@ -418,16 +424,16 @@ class AnthropicMessagesHandler(BaseTranslation): if tool_calls_list: guardrail_inputs["tool_calls"] = tool_calls_list - _guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid - inputs=guardrail_inputs, - request_data=request_data if request_data is not None else {}, - input_type="response", - logging_obj=litellm_logging_obj, + _guardrailed_inputs = ( + await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid + inputs=guardrail_inputs, + request_data=request_data if request_data is not None else {}, + input_type="response", + logging_obj=litellm_logging_obj, + ) ) else: - verbose_proxy_logger.debug( - "Skipping output guardrail - model response has no choices" - ) + verbose_proxy_logger.debug("Skipping output guardrail - model response has no choices") return responses_so_far string_so_far = self.get_streaming_string_so_far(responses_so_far) @@ -454,9 +460,7 @@ class AnthropicMessagesHandler(BaseTranslation): request_data[key] = response if "litellm_metadata" not in request_data: - user_metadata = self.transform_user_api_key_dict_to_metadata( - user_api_key_dict - ) + user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict) if user_metadata: request_data["litellm_metadata"] = user_metadata return request_data @@ -604,9 +608,7 @@ class AnthropicMessagesHandler(BaseTranslation): if delta.get("type") == "text_delta": text += delta.get("text", "") except json.JSONDecodeError: - verbose_proxy_logger.warning( - f"Failed to parse JSON from SSE data: {data_line}" - ) + verbose_proxy_logger.warning(f"Failed to parse JSON from SSE data: {data_line}") except Exception as e: verbose_proxy_logger.error(f"Error extracting text from SSE: {e}") @@ -670,14 +672,10 @@ class AnthropicMessagesHandler(BaseTranslation): if stop_reason is not None: return True except json.JSONDecodeError: - verbose_proxy_logger.warning( - f"Failed to parse JSON from SSE data: {data_line}" - ) + verbose_proxy_logger.warning(f"Failed to parse JSON from SSE data: {data_line}") except Exception as e: - verbose_proxy_logger.error( - f"Error checking streaming end in SSE: {e}" - ) + verbose_proxy_logger.error(f"Error checking streaming end in SSE: {e}") # Handle already-parsed dict format elif isinstance(response, dict): @@ -783,10 +781,7 @@ class AnthropicMessagesHandler(BaseTranslation): if isinstance(content_block, dict): if content_block.get("type") == "text": cast(Dict[str, Any], content_block)["text"] = guardrail_response - elif ( - hasattr(content_block, "type") - and getattr(content_block, "type", None) == "text" - ): + elif hasattr(content_block, "type") and getattr(content_block, "type", None) == "text": # Update Pydantic object's text attribute if hasattr(content_block, "text"): content_block.text = guardrail_response diff --git a/litellm/utils.py b/litellm/utils.py index e0aa8575473..85fc37132b5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -217,16 +217,12 @@ _CALL_TYPE_ENUM_MAP: dict = {ct.value: ct for ct in CallTypes} try: # Python 3.9+ - with ( - resources.files("litellm.litellm_core_utils.tokenizers") - .joinpath("anthropic_tokenizer.json") - .open("r", encoding="utf-8") as f - ): + with resources.files("litellm.litellm_core_utils.tokenizers").joinpath("anthropic_tokenizer.json").open( + "r", encoding="utf-8" + ) as f: json_data = json.load(f) except (ImportError, AttributeError, TypeError): - with resources.open_text( - "litellm.litellm_core_utils.tokenizers", "anthropic_tokenizer.json" - ) as f: + with resources.open_text("litellm.litellm_core_utils.tokenizers", "anthropic_tokenizer.json") as f: json_data = json.load(f) # Convert to str (if necessary) @@ -509,9 +505,7 @@ def custom_llm_setup(): litellm._custom_providers.append(custom_llm["provider"]) -def _add_custom_logger_callback_to_specific_event( - callback: str, logging_event: Literal["success", "failure"] -) -> None: +def _add_custom_logger_callback_to_specific_event(callback: str, logging_event: Literal["success", "failure"]) -> None: """ Add a custom logger callback to the specific event """ @@ -533,44 +527,20 @@ def _add_custom_logger_callback_to_specific_event( ) if callback_class: - if ( - logging_event == "success" - and _custom_logger_class_exists_in_success_callbacks(callback_class) - is False - ): - litellm.logging_callback_manager.add_litellm_success_callback( - callback_class - ) - litellm.logging_callback_manager.add_litellm_async_success_callback( - callback_class - ) + if logging_event == "success" and _custom_logger_class_exists_in_success_callbacks(callback_class) is False: + litellm.logging_callback_manager.add_litellm_success_callback(callback_class) + litellm.logging_callback_manager.add_litellm_async_success_callback(callback_class) if callback in litellm.success_callback: - litellm.success_callback.remove( - callback - ) # remove the string from the callback list + litellm.success_callback.remove(callback) # remove the string from the callback list if callback in litellm._async_success_callback: - litellm._async_success_callback.remove( - callback - ) # remove the string from the callback list - elif ( - logging_event == "failure" - and _custom_logger_class_exists_in_failure_callbacks(callback_class) - is False - ): - litellm.logging_callback_manager.add_litellm_failure_callback( - callback_class - ) - litellm.logging_callback_manager.add_litellm_async_failure_callback( - callback_class - ) + litellm._async_success_callback.remove(callback) # remove the string from the callback list + elif logging_event == "failure" and _custom_logger_class_exists_in_failure_callbacks(callback_class) is False: + litellm.logging_callback_manager.add_litellm_failure_callback(callback_class) + litellm.logging_callback_manager.add_litellm_async_failure_callback(callback_class) if callback in litellm.failure_callback: - litellm.failure_callback.remove( - callback - ) # remove the string from the callback list + litellm.failure_callback.remove(callback) # remove the string from the callback list if callback in litellm._async_failure_callback: - litellm._async_failure_callback.remove( - callback - ) # remove the string from the callback list + litellm._async_failure_callback.remove(callback) # remove the string from the callback list def _custom_logger_class_exists_in_success_callbacks( @@ -584,8 +554,7 @@ def _custom_logger_class_exists_in_success_callbacks( Prevents double adding a custom logger callback to the litellm callbacks """ return any( - isinstance(cb, type(callback_class)) - for cb in litellm.success_callback + litellm._async_success_callback + isinstance(cb, type(callback_class)) for cb in litellm.success_callback + litellm._async_success_callback ) @@ -600,8 +569,7 @@ def _custom_logger_class_exists_in_failure_callbacks( Prevents double adding a custom logger callback to the litellm callbacks """ return any( - isinstance(cb, type(callback_class)) - for cb in litellm.failure_callback + litellm._async_failure_callback + isinstance(cb, type(callback_class)) for cb in litellm.failure_callback + litellm._async_failure_callback ) @@ -682,9 +650,7 @@ def _remove_thought_signature_from_id(tool_call_id: str, separator: str) -> str: return tool_call_id -def _process_assistant_message_tool_calls( - msg_copy: dict, thought_signature_separator: str -) -> dict: +def _process_assistant_message_tool_calls(msg_copy: dict, thought_signature_separator: str) -> dict: """ Process assistant message to remove thought signatures from tool call IDs. """ @@ -707,9 +673,7 @@ def _process_assistant_message_tool_calls( # Remove thought signature from ID if present if isinstance(tc_dict.get("id"), str): if thought_signature_separator in tc_dict["id"]: - tc_dict["id"] = _remove_thought_signature_from_id( - tc_dict["id"], thought_signature_separator - ) + tc_dict["id"] = _remove_thought_signature_from_id(tc_dict["id"], thought_signature_separator) new_tool_calls.append(tc_dict) msg_copy["tool_calls"] = new_tool_calls @@ -730,9 +694,7 @@ def _process_tool_message_id(msg_copy: dict, thought_signature_separator: str) - return msg_copy -def _remove_thought_signatures_from_messages( - messages: List, thought_signature_separator: str -) -> List: +def _remove_thought_signatures_from_messages(messages: List, thought_signature_separator: str) -> List: """ Remove thought signatures from tool call IDs in all messages. """ @@ -750,9 +712,7 @@ def _remove_thought_signatures_from_messages( continue # Process assistant messages with tool_calls - msg_dict = _process_assistant_message_tool_calls( - msg_dict, thought_signature_separator - ) + msg_dict = _process_assistant_message_tool_calls(msg_dict, thought_signature_separator) # Process tool messages with tool_call_id msg_dict = _process_tool_message_id(msg_dict, thought_signature_separator) @@ -783,15 +743,11 @@ def function_setup( function_id: Optional[str] = kwargs["id"] if "id" in kwargs else None ## LAZY LOAD COROUTINE CHECKER ## - get_coroutine_checker_fn = getattr( - sys.modules[__name__], "get_coroutine_checker" - ) + get_coroutine_checker_fn = getattr(sys.modules[__name__], "get_coroutine_checker") coroutine_checker = get_coroutine_checker_fn() ## DYNAMIC CALLBACKS ## - dynamic_callbacks: Optional[List[Union[str, Callable, "CustomLogger"]]] = ( - kwargs.pop("callbacks", None) - ) + dynamic_callbacks: Optional[List[Union[str, Callable, "CustomLogger"]]] = kwargs.pop("callbacks", None) all_callbacks = get_dynamic_callbacks(dynamic_callbacks=dynamic_callbacks) if len(all_callbacks) > 0: @@ -804,36 +760,23 @@ def function_setup( llm_router=None, # type: ignore ) if callback is None or any( - isinstance(cb, type(callback)) - for cb in litellm._async_success_callback + isinstance(cb, type(callback)) for cb in litellm._async_success_callback ): # don't double add a callback continue if callback not in litellm.input_callback: litellm.input_callback.append(callback) # type: ignore if callback not in litellm.success_callback: - litellm.logging_callback_manager.add_litellm_success_callback( - callback - ) # type: ignore + litellm.logging_callback_manager.add_litellm_success_callback(callback) # type: ignore if callback not in litellm.failure_callback: - litellm.logging_callback_manager.add_litellm_failure_callback( - callback - ) # type: ignore + litellm.logging_callback_manager.add_litellm_failure_callback(callback) # type: ignore if callback not in litellm._async_success_callback: - litellm.logging_callback_manager.add_litellm_async_success_callback( - callback - ) # type: ignore + litellm.logging_callback_manager.add_litellm_async_success_callback(callback) # type: ignore if callback not in litellm._async_failure_callback: - litellm.logging_callback_manager.add_litellm_async_failure_callback( - callback - ) # type: ignore - print_verbose( - f"Initialized litellm callbacks, Async Success Callbacks: {litellm._async_success_callback}" - ) + litellm.logging_callback_manager.add_litellm_async_failure_callback(callback) # type: ignore + print_verbose(f"Initialized litellm callbacks, Async Success Callbacks: {litellm._async_success_callback}") if ( - len(litellm.input_callback) > 0 - or len(litellm.success_callback) > 0 - or len(litellm.failure_callback) > 0 + len(litellm.input_callback) > 0 or len(litellm.success_callback) > 0 or len(litellm.failure_callback) > 0 ) and len( callback_list # type: ignore ) == 0: # type: ignore @@ -861,21 +804,14 @@ def function_setup( removed_async_items = [] for index, callback in enumerate(litellm.success_callback): # type: ignore if coroutine_checker.is_async_callable(callback): - litellm.logging_callback_manager.add_litellm_async_success_callback( - callback - ) + litellm.logging_callback_manager.add_litellm_async_success_callback(callback) removed_async_items.append(index) elif callback == "dynamodb" or callback == "openmeter": # dynamo is an async callback, it's used for the proxy and needs to be async # we only support async dynamo db logging for acompletion/aembedding since that's used on proxy - litellm.logging_callback_manager.add_litellm_async_success_callback( - callback - ) + litellm.logging_callback_manager.add_litellm_async_success_callback(callback) removed_async_items.append(index) - elif ( - callback in litellm._known_custom_logger_compatible_callbacks - and isinstance(callback, str) - ): + elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str): _add_custom_logger_callback_to_specific_event(callback, "success") # Pop the async items from success_callback in reverse order to avoid index issues @@ -886,42 +822,23 @@ def function_setup( removed_async_items = [] for index, callback in enumerate(litellm.failure_callback): # type: ignore if coroutine_checker.is_async_callable(callback): - litellm.logging_callback_manager.add_litellm_async_failure_callback( - callback - ) + litellm.logging_callback_manager.add_litellm_async_failure_callback(callback) removed_async_items.append(index) - elif ( - callback in litellm._known_custom_logger_compatible_callbacks - and isinstance(callback, str) - ): + elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str): _add_custom_logger_callback_to_specific_event(callback, "failure") # Pop the async items from failure_callback in reverse order to avoid index issues for index in reversed(removed_async_items): litellm.failure_callback.pop(index) ### DYNAMIC CALLBACKS ### - dynamic_success_callbacks: Optional[ - List[Union[str, Callable, "CustomLogger"]] - ] = None - dynamic_async_success_callbacks: Optional[ - List[Union[str, Callable, "CustomLogger"]] - ] = None - dynamic_failure_callbacks: Optional[ - List[Union[str, Callable, "CustomLogger"]] - ] = None - dynamic_async_failure_callbacks: Optional[ - List[Union[str, Callable, "CustomLogger"]] - ] = None - if kwargs.get("success_callback", None) is not None and isinstance( - kwargs["success_callback"], list - ): + dynamic_success_callbacks: Optional[List[Union[str, Callable, "CustomLogger"]]] = None + dynamic_async_success_callbacks: Optional[List[Union[str, Callable, "CustomLogger"]]] = None + dynamic_failure_callbacks: Optional[List[Union[str, Callable, "CustomLogger"]]] = None + dynamic_async_failure_callbacks: Optional[List[Union[str, Callable, "CustomLogger"]]] = None + if kwargs.get("success_callback", None) is not None and isinstance(kwargs["success_callback"], list): removed_async_items = [] for index, callback in enumerate(kwargs["success_callback"]): - if ( - coroutine_checker.is_async_callable(callback) - or callback == "dynamodb" - or callback == "s3" - ): + if coroutine_checker.is_async_callable(callback) or callback == "dynamodb" or callback == "s3": if dynamic_async_success_callbacks is not None and isinstance( dynamic_async_success_callbacks, list ): @@ -933,9 +850,7 @@ def function_setup( for index in reversed(removed_async_items): kwargs["success_callback"].pop(index) dynamic_success_callbacks = kwargs.pop("success_callback") - if kwargs.get("failure_callback", None) is not None and isinstance( - kwargs["failure_callback"], list - ): + if kwargs.get("failure_callback", None) is not None and isinstance(kwargs["failure_callback"], list): dynamic_failure_callbacks = kwargs.pop("failure_callback") if add_breadcrumb: @@ -1020,9 +935,7 @@ def function_setup( # Only process if target is NOT a Gemini model if not _is_gemini_model(model, custom_llm_provider): - verbose_logger.debug( - "Removing thought signatures from tool call IDs for non-Gemini model" - ) + verbose_logger.debug("Removing thought signatures from tool call IDs for non-Gemini model") # Process messages to remove thought signatures processed_messages = _remove_thought_signatures_from_messages( @@ -1039,37 +952,18 @@ def function_setup( except Exception as e: # Log the error but don't fail the request - verbose_logger.warning( - f"Error removing thought signatures from tool call IDs: {str(e)}" - ) - elif ( - call_type == CallTypes.embedding.value - or call_type == CallTypes.aembedding.value - ): + verbose_logger.warning(f"Error removing thought signatures from tool call IDs: {str(e)}") + elif call_type == CallTypes.embedding.value or call_type == CallTypes.aembedding.value: messages = args[1] if len(args) > 1 else kwargs.get("input", None) - elif ( - call_type == CallTypes.image_generation.value - or call_type == CallTypes.aimage_generation.value - ): + elif call_type == CallTypes.image_generation.value or call_type == CallTypes.aimage_generation.value: messages = args[0] if len(args) > 0 else kwargs["prompt"] - elif ( - call_type == CallTypes.moderation.value - or call_type == CallTypes.amoderation.value - ): + elif call_type == CallTypes.moderation.value or call_type == CallTypes.amoderation.value: messages = args[1] if len(args) > 1 else kwargs["input"] - elif ( - call_type == CallTypes.atext_completion.value - or call_type == CallTypes.text_completion.value - ): + elif call_type == CallTypes.atext_completion.value or call_type == CallTypes.text_completion.value: messages = args[0] if len(args) > 0 else kwargs["prompt"] - elif ( - call_type == CallTypes.rerank.value or call_type == CallTypes.arerank.value - ): + elif call_type == CallTypes.rerank.value or call_type == CallTypes.arerank.value: messages = kwargs.get("query") - elif ( - call_type == CallTypes.atranscription.value - or call_type == CallTypes.transcription.value - ): + elif call_type == CallTypes.atranscription.value or call_type == CallTypes.transcription.value: _file_obj: FileTypes = args[1] if len(args) > 1 else kwargs["file"] # Lazy import audio_utils.utils only when needed for transcription calls audio_utils = _get_cached_audio_utils() @@ -1079,20 +973,12 @@ def function_setup( else: kwargs["metadata"] = {"file_checksum": file_checksum} messages = file_checksum - elif ( - call_type == CallTypes.aspeech.value or call_type == CallTypes.speech.value - ): + elif call_type == CallTypes.aspeech.value or call_type == CallTypes.speech.value: messages = kwargs.get("input", "speech") - elif ( - call_type == CallTypes.aresponses.value - or call_type == CallTypes.responses.value - ): + elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value: # Handle both 'input' (standard Responses API) and 'messages' (Cursor chat format) messages = ( - args[0] - if len(args) > 0 - else kwargs.get("input") - or kwargs.get("messages", "default-message-value") + args[0] if len(args) > 0 else kwargs.get("input") or kwargs.get("messages", "default-message-value") ) elif ( call_type == CallTypes.generate_content.value @@ -1119,16 +1005,11 @@ def function_setup( config=kwargs.get("config"), ) transformed_messages = transformed.get("messages", []) - messages = ( - get_last_user_message(transformed_messages) - or "default-message-value" - ) + messages = get_last_user_message(transformed_messages) or "default-message-value" else: messages = "default-message-value" except Exception as e: - verbose_logger.debug( - f"Error extracting messages from Google contents: {str(e)}" - ) + verbose_logger.debug(f"Error extracting messages from Google contents: {str(e)}") messages = "default-message-value" else: messages = "default-message-value" @@ -1138,9 +1019,7 @@ def function_setup( call_type=call_type, ): stream = True - get_litellm_logging_class = getattr( - sys.modules[__name__], "get_litellm_logging_class" - ) + get_litellm_logging_class = getattr(sys.modules[__name__], "get_litellm_logging_class") logging_obj = get_litellm_logging_class()( # Victim for object pool model=model, # type: ignore messages=messages, @@ -1162,16 +1041,16 @@ def function_setup( litellm_params: Dict[str, Any] = {"api_base": ""} if "metadata" in kwargs: litellm_params["metadata"] = kwargs["metadata"] - if "litellm_metadata" in kwargs and isinstance( - kwargs["litellm_metadata"], dict - ): + if "litellm_metadata" in kwargs and isinstance(kwargs["litellm_metadata"], dict): litellm_params["litellm_metadata"] = kwargs["litellm_metadata"].copy() # For endpoints like /v1/messages that use "litellm_metadata" instead # of "metadata" (to avoid conflicting with provider API metadata fields), # populate litellm_params["metadata"] so callbacks (e.g. Langfuse) that # read API key info from litellm_params["metadata"] see the fields. + # Use a reference (not a copy) so that guardrails writing to + # litellm_metadata after this point are visible in standard logging. if not litellm_params.get("metadata"): - litellm_params["metadata"] = kwargs["litellm_metadata"].copy() + litellm_params["metadata"] = kwargs["litellm_metadata"] logging_obj.update_environment_variables( model=model, @@ -1182,9 +1061,7 @@ def function_setup( ) return logging_obj, kwargs except Exception as e: - verbose_logger.exception( - "litellm.utils.py::function_setup() - [Non-Blocking] Error in function_setup" - ) + verbose_logger.exception("litellm.utils.py::function_setup() - [Non-Blocking] Error in function_setup") raise e @@ -1207,9 +1084,7 @@ async def _client_async_logging_helper( from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - async_coroutine=logging_obj.async_success_handler( - result=result, start_time=start_time, end_time=end_time - ) + async_coroutine=logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time) ) ################################################ @@ -1222,9 +1097,7 @@ async def _client_async_logging_helper( ) -def _get_wrapper_num_retries( - kwargs: Dict[str, Any], exception: Exception -) -> Tuple[Optional[int], Dict[str, Any]]: +def _get_wrapper_num_retries(kwargs: Dict[str, Any], exception: Exception) -> Tuple[Optional[int], Dict[str, Any]]: """ Get the number of retries from the kwargs and the retry policy. Used for the wrapper functions. @@ -1234,9 +1107,7 @@ def _get_wrapper_num_retries( if num_retries is None: num_retries = litellm.num_retries if kwargs.get("retry_policy", None): - get_num_retries_from_retry_policy = getattr( - sys.modules[__name__], "get_num_retries_from_retry_policy" - ) + get_num_retries_from_retry_policy = getattr(sys.modules[__name__], "get_num_retries_from_retry_policy") reset_retry_policy = getattr(sys.modules[__name__], "reset_retry_policy") retry_policy_num_retries = get_num_retries_from_retry_policy( exception=exception, @@ -1249,17 +1120,13 @@ def _get_wrapper_num_retries( return num_retries, kwargs -def _get_wrapper_timeout( - kwargs: Dict[str, Any], exception: Exception -) -> Optional[Union[float, int, httpx.Timeout]]: +def _get_wrapper_timeout(kwargs: Dict[str, Any], exception: Exception) -> Optional[Union[float, int, httpx.Timeout]]: """ Get the timeout from the kwargs Used for the wrapper functions. """ - timeout = cast( - Optional[Union[float, int, httpx.Timeout]], kwargs.get("timeout", None) - ) + timeout = cast(Optional[Union[float, int, httpx.Timeout]], kwargs.get("timeout", None)) return timeout @@ -1285,9 +1152,7 @@ async def async_pre_call_deployment_hook(kwargs: Dict[str, Any], call_type: str) CustomLogger = _get_cached_custom_logger() for callback in litellm.callbacks: if isinstance(callback, CustomLogger): - result = await callback.async_pre_call_deployment_hook( - modified_kwargs, typed_call_type - ) + result = await callback.async_pre_call_deployment_hook(modified_kwargs, typed_call_type) if result is not None: modified_kwargs = result @@ -1329,21 +1194,13 @@ def post_call_processing( pass else: call_type = original_function.__name__ - if ( - call_type == CallTypes.completion.value - or call_type == CallTypes.acompletion.value - ): + if call_type == CallTypes.completion.value or call_type == CallTypes.acompletion.value: is_coroutine = check_coroutine(original_response) if is_coroutine is True: pass else: - if ( - isinstance(original_response, ModelResponse) - and len(original_response.choices) > 0 - ): - model_response: Optional[str] = original_response.choices[ - 0 - ].message.content # type: ignore + if isinstance(original_response, ModelResponse) and len(original_response.choices) > 0: + model_response: Optional[str] = original_response.choices[0].message.content # type: ignore if model_response is not None: ### POST-CALL RULES ### rules_obj.post_call_rules(input=model_response, model=model) @@ -1364,8 +1221,7 @@ def post_call_processing( if ( optional_params is not None and "response_format" in optional_params - and optional_params["response_format"] - is not None + and optional_params["response_format"] is not None ): json_response_format: Optional[dict] = None if ( @@ -1373,29 +1229,18 @@ def post_call_processing( optional_params["response_format"], dict, ) - and optional_params["response_format"].get( - "json_schema" - ) - is not None + and optional_params["response_format"].get("json_schema") is not None ): - json_response_format = optional_params[ - "response_format" - ] + json_response_format = optional_params["response_format"] elif _parsing._completions.is_basemodel_type( optional_params["response_format"] # type: ignore ): - json_response_format = ( - type_to_response_format_param( - response_format=optional_params[ - "response_format" - ] - ) + json_response_format = type_to_response_format_param( + response_format=optional_params["response_format"] ) if json_response_format is not None: litellm.litellm_core_utils.json_validation_rule.validate_schema( - schema=json_response_format[ - "json_schema" - ]["schema"], + schema=json_response_format["json_schema"]["schema"], response=model_response, ) except TypeError: @@ -1405,28 +1250,18 @@ def post_call_processing( and "response_format" in optional_params and isinstance(optional_params["response_format"], dict) and "type" in optional_params["response_format"] - and optional_params["response_format"]["type"] - == "json_object" - and "response_schema" - in optional_params["response_format"] + and optional_params["response_format"]["type"] == "json_object" + and "response_schema" in optional_params["response_format"] and isinstance( - optional_params["response_format"][ - "response_schema" - ], + optional_params["response_format"]["response_schema"], dict, ) - and "enforce_validation" - in optional_params["response_format"] - and optional_params["response_format"][ - "enforce_validation" - ] - is True + and "enforce_validation" in optional_params["response_format"] + and optional_params["response_format"]["enforce_validation"] is True ): # schema given, json response expected, and validation enforced litellm.litellm_core_utils.json_validation_rule.validate_schema( - schema=optional_params["response_format"][ - "response_schema" - ], + schema=optional_params["response_format"]["response_schema"], response=model_response, ) @@ -1447,9 +1282,7 @@ def client(original_function): # [OPTIONAL] CHECK MAX RETRIES / REQUEST if litellm.num_retries_per_request is not None: # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] - previous_models = (kwargs.get("metadata") or {}).get( - "previous_models", None - ) + previous_models = (kwargs.get("metadata") or {}).get("previous_models", None) if previous_models is not None: if litellm.num_retries_per_request <= len(previous_models): raise Exception("Max retries per request hit!") @@ -1460,16 +1293,11 @@ def client(original_function): kwargs=kwargs, call_type=call_type, ): - if ( - "complete_response" in kwargs - and kwargs["complete_response"] is True - ): + if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks = [] for idx, chunk in enumerate(result): chunks.append(chunk) - return litellm.stream_chunk_builder( - chunks, messages=kwargs.get("messages", None) - ) + return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: return result @@ -1479,9 +1307,7 @@ def client(original_function): print_args_passed_to_litellm(original_function, args, kwargs) start_time = datetime.datetime.now() result = None - logging_obj: Optional[LiteLLMLoggingObject] = kwargs.get( - "litellm_logging_obj", None - ) + logging_obj: Optional[LiteLLMLoggingObject] = kwargs.get("litellm_logging_obj", None) # only set litellm_call_id if its not in kwargs if "litellm_call_id" not in kwargs: @@ -1491,14 +1317,10 @@ def client(original_function): try: if logging_obj is None: - logging_obj, kwargs = function_setup( - original_function.__name__, rules_obj, start_time, *args, **kwargs - ) + logging_obj, kwargs = function_setup(original_function.__name__, rules_obj, start_time, *args, **kwargs) # Type assertion: logging_obj is guaranteed to be non-None after function_setup - assert logging_obj is not None, ( - "logging_obj should not be None after function_setup" - ) + assert logging_obj is not None, "logging_obj should not be None after function_setup" ## LOAD CREDENTIALS load_credentials_from_list(kwargs) @@ -1522,9 +1344,7 @@ def client(original_function): # [OPTIONAL] CHECK MAX RETRIES / REQUEST if litellm.num_retries_per_request is not None: # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] - previous_models = (kwargs.get("metadata") or {}).get( - "previous_models", None - ) + previous_models = (kwargs.get("metadata") or {}).get("previous_models", None) if previous_models is not None: if litellm.num_retries_per_request <= len(previous_models): raise Exception("Max retries per request hit!") @@ -1537,10 +1357,7 @@ def client(original_function): if ( ( ( - ( - kwargs.get("caching", None) is None - and litellm.cache is not None - ) + (kwargs.get("caching", None) is None and litellm.cache is not None) or kwargs.get("caching", False) is True ) and kwargs.get("cache", {}).get("no-cache", False) is not True @@ -1555,16 +1372,14 @@ def client(original_function): ): # allow users to control returning cached responses from the completion function # checking cache verbose_logger.debug("INSIDE CHECKING SYNC CACHE") - caching_handler_response: "CachingHandlerResponse" = ( - _llm_caching_handler._sync_get_cache( - model=model or "", - original_function=original_function, - logging_obj=logging_obj, - start_time=start_time, - call_type=call_type, - kwargs=kwargs, - args=args, - ) + caching_handler_response: "CachingHandlerResponse" = _llm_caching_handler._sync_get_cache( + model=model or "", + original_function=original_function, + logging_obj=logging_obj, + start_time=start_time, + call_type=call_type, + kwargs=kwargs, + args=args, ) if caching_handler_response.cached_result is not None: @@ -1575,8 +1390,7 @@ def client(original_function): if ( kwargs.get("max_tokens", None) is not None and model is not None - and litellm.modify_params - is True # user is okay with params being modified + and litellm.modify_params is True # user is okay with params being modified and ( call_type == CallTypes.acompletion.value or call_type == CallTypes.completion.value @@ -1611,21 +1425,14 @@ def client(original_function): kwargs=kwargs, call_type=call_type, ): - if ( - "complete_response" in kwargs - and kwargs["complete_response"] is True - ): + if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks = [] for idx, chunk in enumerate(result): chunks.append(chunk) - return litellm.stream_chunk_builder( - chunks, messages=kwargs.get("messages", None) - ) + return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: # RETURN RESULT - update_response_metadata = getattr( - sys.modules[__name__], "update_response_metadata" - ) + update_response_metadata = getattr(sys.modules[__name__], "update_response_metadata") update_response_metadata( result=result, logging_obj=logging_obj, @@ -1678,9 +1485,7 @@ def client(original_function): end_time, ) # RETURN RESULT - update_response_metadata = getattr( - sys.modules[__name__], "update_response_metadata" - ) + update_response_metadata = getattr(sys.modules[__name__], "update_response_metadata") update_response_metadata( result=result, logging_obj=logging_obj, @@ -1693,29 +1498,19 @@ def client(original_function): except Exception as e: call_type = original_function.__name__ if call_type == CallTypes.completion.value: - num_retries = ( - kwargs.get("num_retries", None) or litellm.num_retries or None - ) + num_retries = kwargs.get("num_retries", None) or litellm.num_retries or None if kwargs.get("retry_policy", None): get_num_retries_from_retry_policy = getattr( sys.modules[__name__], "get_num_retries_from_retry_policy" ) - reset_retry_policy = getattr( - sys.modules[__name__], "reset_retry_policy" - ) + reset_retry_policy = getattr(sys.modules[__name__], "reset_retry_policy") num_retries = get_num_retries_from_retry_policy( exception=e, retry_policy=kwargs.get("retry_policy"), ) - kwargs["retry_policy"] = ( - reset_retry_policy() - ) # prevent infinite loops - litellm.num_retries = ( - None # set retries to None to prevent infinite loops - ) - context_window_fallback_dict = kwargs.get( - "context_window_fallback_dict", {} - ) + kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops + litellm.num_retries = None # set retries to None to prevent infinite loops + context_window_fallback_dict = kwargs.get("context_window_fallback_dict", {}) _is_litellm_router_call = "model_group" in ( kwargs.get("metadata") or {} @@ -1742,26 +1537,18 @@ def client(original_function): kwargs["model"] = context_window_fallback_dict[model] return original_function(*args, **kwargs) elif call_type == CallTypes.responses.value: - num_retries = ( - kwargs.get("num_retries", None) or litellm.num_retries or None - ) + num_retries = kwargs.get("num_retries", None) or litellm.num_retries or None if kwargs.get("retry_policy", None): get_num_retries_from_retry_policy = getattr( sys.modules[__name__], "get_num_retries_from_retry_policy" ) - reset_retry_policy = getattr( - sys.modules[__name__], "reset_retry_policy" - ) + reset_retry_policy = getattr(sys.modules[__name__], "reset_retry_policy") num_retries = get_num_retries_from_retry_policy( exception=e, retry_policy=kwargs.get("retry_policy"), ) - kwargs["retry_policy"] = ( - reset_retry_policy() - ) # prevent infinite loops - litellm.num_retries = ( - None # set retries to None to prevent infinite loops - ) + kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops + litellm.num_retries = None # set retries to None to prevent infinite loops _is_litellm_router_call = "model_group" in ( kwargs.get("metadata") or {} @@ -1791,12 +1578,8 @@ def client(original_function): print_args_passed_to_litellm(original_function, args, kwargs) start_time = datetime.datetime.now() result = None - _update_response_metadata = getattr( - sys.modules[__name__], "update_response_metadata" - ) - logging_obj: Optional[LiteLLMLoggingObject] = kwargs.get( - "litellm_logging_obj", None - ) + _update_response_metadata = getattr(sys.modules[__name__], "update_response_metadata") + logging_obj: Optional[LiteLLMLoggingObject] = kwargs.get("litellm_logging_obj", None) LLMCachingHandler = _get_cached_llm_caching_handler() _llm_caching_handler: "LLMCachingHandler" = LLMCachingHandler( original_function=original_function, @@ -1815,14 +1598,10 @@ def client(original_function): try: if logging_obj is None: - logging_obj, kwargs = function_setup( - original_function.__name__, rules_obj, start_time, *args, **kwargs - ) + logging_obj, kwargs = function_setup(original_function.__name__, rules_obj, start_time, *args, **kwargs) # Type assertion: logging_obj is guaranteed to be non-None after function_setup - assert logging_obj is not None, ( - "logging_obj should not be None after function_setup" - ) + assert logging_obj is not None, "logging_obj should not be None after function_setup" modified_kwargs = await async_pre_call_deployment_hook(kwargs, call_type) if modified_kwargs is not None: @@ -1850,23 +1629,20 @@ def client(original_function): print_verbose( f"ASYNC kwargs[caching]: {kwargs.get('caching', False)}; litellm.cache: {litellm.cache}; kwargs.get('cache'): {kwargs.get('cache', None)}" ) - _caching_handler_response: "Optional[CachingHandlerResponse]" = ( - await _llm_caching_handler._async_get_cache( - model=model or "", - original_function=original_function, - logging_obj=logging_obj, - start_time=start_time, - call_type=call_type, - kwargs=kwargs, - args=args, - ) + _caching_handler_response: "Optional[CachingHandlerResponse]" = await _llm_caching_handler._async_get_cache( + model=model or "", + original_function=original_function, + logging_obj=logging_obj, + start_time=start_time, + call_type=call_type, + kwargs=kwargs, + args=args, ) if _caching_handler_response is not None: if ( _caching_handler_response.cached_result is not None - and _caching_handler_response.final_embedding_cached_response - is None + and _caching_handler_response.final_embedding_cached_response is None ): return _caching_handler_response.cached_result @@ -1877,8 +1653,7 @@ def client(original_function): if ( kwargs.get("max_tokens", None) is not None and model is not None - and litellm.modify_params - is True # user is okay with params being modified + and litellm.modify_params is True # user is okay with params being modified and ( call_type == CallTypes.acompletion.value or call_type == CallTypes.completion.value @@ -1915,16 +1690,11 @@ def client(original_function): kwargs=kwargs, call_type=call_type, ): - if ( - "complete_response" in kwargs - and kwargs["complete_response"] is True - ): + if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks = [] for idx, chunk in enumerate(result): chunks.append(chunk) - return litellm.stream_chunk_builder( - chunks, messages=kwargs.get("messages", None) - ) + return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: _update_response_metadata( result=result, @@ -2002,8 +1772,7 @@ def client(original_function): if ( isinstance(result, EmbeddingResponse) and _caching_handler_response is not None - and _caching_handler_response.final_embedding_cached_response - is not None + and _caching_handler_response.final_embedding_cached_response is not None ): return _llm_caching_handler._combine_cached_embedding_response_with_api_result( _caching_handler_response=_caching_handler_response, @@ -2033,18 +1802,14 @@ def client(original_function): except Exception as e: raise e try: - await logging_obj.async_failure_handler( - e, traceback_exception, start_time, end_time - ) + await logging_obj.async_failure_handler(e, traceback_exception, start_time, end_time) except Exception as e: raise e call_type = original_function.__name__ num_retries, kwargs = _get_wrapper_num_retries(kwargs=kwargs, exception=e) if call_type == CallTypes.acompletion.value: - context_window_fallback_dict = kwargs.get( - "context_window_fallback_dict", {} - ) + context_window_fallback_dict = kwargs.get("context_window_fallback_dict", {}) _is_litellm_router_call = "model_group" in ( kwargs.get("metadata") or {} @@ -2054,14 +1819,10 @@ def client(original_function): num_retries and not _is_litellm_router_call ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: - litellm.num_retries = ( - None # set retries to None to prevent infinite loops - ) + litellm.num_retries = None # set retries to None to prevent infinite loops kwargs["num_retries"] = num_retries kwargs["original_function"] = original_function - if isinstance( - e, openai.RateLimitError - ): # rate limiting specific error + if isinstance(e, openai.RateLimitError): # rate limiting specific error kwargs["retry_strategy"] = "exponential_backoff_retry" elif isinstance(e, openai.APIError): # generic api error kwargs["retry_strategy"] = "constant_retry" @@ -2088,14 +1849,10 @@ def client(original_function): num_retries and not _is_litellm_router_call ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: - litellm.num_retries = ( - None # set retries to None to prevent infinite loops - ) + litellm.num_retries = None # set retries to None to prevent infinite loops kwargs["num_retries"] = num_retries kwargs["original_function"] = original_function - if isinstance( - e, openai.RateLimitError - ): # rate limiting specific error + if isinstance(e, openai.RateLimitError): # rate limiting specific error kwargs["retry_strategy"] = "exponential_backoff_retry" elif isinstance(e, openai.APIError): # generic api error kwargs["retry_strategy"] = "constant_retry" @@ -2103,9 +1860,7 @@ def client(original_function): except Exception: pass - setattr( - e, "num_retries", num_retries - ) ## IMPORTANT: returns the deployment's num_retries to the router + setattr(e, "num_retries", num_retries) ## IMPORTANT: returns the deployment's num_retries to the router timeout = _get_wrapper_timeout(kwargs=kwargs, exception=e) setattr(e, "timeout", timeout) @@ -2178,9 +1933,7 @@ def _is_streaming_request( return call_type in _STREAMING_CALL_TYPES -def _select_tokenizer( - model: str, custom_tokenizer: Optional[CustomHuggingfaceTokenizer] = None -): +def _select_tokenizer(model: str, custom_tokenizer: Optional[CustomHuggingfaceTokenizer] = None): if custom_tokenizer is not None: _tokenizer = create_pretrained_tokenizer( identifier=custom_tokenizer["identifier"], @@ -2214,9 +1967,7 @@ def _return_openai_tokenizer(model: str) -> SelectTokenizerResponse: def _return_huggingface_tokenizer(model: str) -> Optional[SelectTokenizerResponse]: if model in litellm.cohere_models and "command-r" in model: # cohere - cohere_tokenizer = Tokenizer.from_pretrained( - "Xenova/c4ai-command-r-v01-tokenizer" - ) + cohere_tokenizer = Tokenizer.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") return {"type": "huggingface_tokenizer", "tokenizer": cohere_tokenizer} # anthropic elif model in litellm.anthropic_models and "claude-3" not in model: @@ -2275,38 +2026,28 @@ def decode( tokenizer_json = custom_tokenizer or _select_tokenizer(model=model) if tokenizer_json["type"] == "huggingface_tokenizer": if skip_special_tokens: - tokens = _strip_huggingface_special_token_ids( - tokenizer_json["tokenizer"], tokens - ) - dec = tokenizer_json["tokenizer"].decode( - tokens, skip_special_tokens=skip_special_tokens - ) + tokens = _strip_huggingface_special_token_ids(tokenizer_json["tokenizer"], tokens) + dec = tokenizer_json["tokenizer"].decode(tokens, skip_special_tokens=skip_special_tokens) return dec dec = tokenizer_json["tokenizer"].decode(tokens) return dec -def _strip_huggingface_special_token_ids( - tokenizer: Tokenizer, tokens: List[int] -) -> List[int]: +def _strip_huggingface_special_token_ids(tokenizer: Tokenizer, tokens: List[int]) -> List[int]: try: added_tokens_decoder = tokenizer.get_added_tokens_decoder() except Exception: return tokens special_token_ids = { - token_id - for token_id, added_token in added_tokens_decoder.items() - if getattr(added_token, "special", False) + token_id for token_id, added_token in added_tokens_decoder.items() if getattr(added_token, "special", False) } if not special_token_ids: return tokens return [token for token in tokens if token not in special_token_ids] -def create_pretrained_tokenizer( - identifier: str, revision="main", auth_token: Optional[str] = None -): +def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: Optional[str] = None): """ Creates a tokenizer from an existing file on a HuggingFace repository to be used with `token_counter`. @@ -2326,9 +2067,7 @@ def create_pretrained_tokenizer( auth_token=auth_token, # type: ignore ) except Exception as e: - verbose_logger.error( - f"Error creating pretrained tokenizer: {e}. Defaulting to version without 'auth_token'." - ) + verbose_logger.error(f"Error creating pretrained tokenizer: {e}. Defaulting to version without 'auth_token'.") tokenizer = Tokenizer.from_pretrained(identifier, revision=revision) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @@ -2481,9 +2220,7 @@ def supports_native_streaming(model: str, custom_llm_provider: Optional[str]) -> model=model, custom_llm_provider=custom_llm_provider ) - model_info = _get_model_info_helper( - model=model, custom_llm_provider=custom_llm_provider - ) + model_info = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) supports_native_streaming = model_info.get("supports_native_streaming", True) if supports_native_streaming is None: supports_native_streaming = True @@ -2495,9 +2232,7 @@ def supports_native_streaming(model: str, custom_llm_provider: Optional[str]) -> return False -def supports_response_schema( - model: str, custom_llm_provider: Optional[str] = None -) -> bool: +def supports_response_schema(model: str, custom_llm_provider: Optional[str] = None) -> bool: """ Check if the given model + provider supports 'response_schema' as a param. @@ -2513,9 +2248,7 @@ def supports_response_schema( ## GET LLM PROVIDER ## try: get_llm_provider = getattr(sys.modules[__name__], "get_llm_provider") - model, custom_llm_provider, _, _ = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider - ) + model, custom_llm_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) except Exception as e: verbose_logger.debug( f"Model not found or error in checking response schema support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {str(e)}" @@ -2540,9 +2273,7 @@ def supports_response_schema( ) -def supports_parallel_function_calling( - model: str, custom_llm_provider: Optional[str] = None -) -> bool: +def supports_parallel_function_calling(model: str, custom_llm_provider: Optional[str] = None) -> bool: """ Check if the given model supports parallel tool calls and return a boolean value. """ @@ -2553,9 +2284,7 @@ def supports_parallel_function_calling( ) -def supports_function_calling( - model: str, custom_llm_provider: Optional[str] = None -) -> bool: +def supports_function_calling(model: str, custom_llm_provider: Optional[str] = None) -> bool: """ Check if the given model supports function calling and return a boolean value. @@ -2580,9 +2309,7 @@ def supports_tool_choice(model: str, custom_llm_provider: Optional[str] = None) """ Check if the given model supports `tool_choice` and return a boolean value. """ - return _supports_factory( - model=model, custom_llm_provider=custom_llm_provider, key="supports_tool_choice" - ) + return _supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_tool_choice") def _supports_provider_info_factory( @@ -2592,9 +2319,7 @@ def _supports_provider_info_factory( Check if the given model supports a provider specific model info and return a boolean value. """ - provider_info = get_provider_info( - model=model, custom_llm_provider=custom_llm_provider - ) + provider_info = get_provider_info(model=model, custom_llm_provider=custom_llm_provider) if provider_info is not None and provider_info.get(key, False) is True: return True @@ -2620,9 +2345,7 @@ def _supports_factory(model: str, custom_llm_provider: Optional[str], key: str) model=model, custom_llm_provider=custom_llm_provider ) - model_info = _get_model_info_helper( - model=model, custom_llm_provider=custom_llm_provider - ) + model_info = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) if model_info.get(key, False) is True: return True @@ -2637,9 +2360,7 @@ def _supports_factory(model: str, custom_llm_provider: Optional[str], key: str) if bare_entry.get(key, False) is True: return True - supported_by_provider = _supports_provider_info_factory( - model, custom_llm_provider, key - ) + supported_by_provider = _supports_provider_info_factory(model, custom_llm_provider, key) if supported_by_provider is not None: return supported_by_provider @@ -2649,18 +2370,14 @@ def _supports_factory(model: str, custom_llm_provider: Optional[str], key: str) f"Model not found or error in checking {key} support. You passed model={model}, custom_llm_provider={custom_llm_provider}. Error: {str(e)}" ) - supported_by_provider = _supports_provider_info_factory( - model, custom_llm_provider, key - ) + supported_by_provider = _supports_provider_info_factory(model, custom_llm_provider, key) if supported_by_provider is not None: return supported_by_provider return False -def _is_explicitly_disabled_factory( - model: str, custom_llm_provider: Optional[str], key: str -) -> bool: +def _is_explicitly_disabled_factory(model: str, custom_llm_provider: Optional[str], key: str) -> bool: """Return True only when the model map explicitly sets *key* to ``False``. This is the opt-out mirror of :func:`_supports_factory`. Where @@ -2677,9 +2394,7 @@ def _is_explicitly_disabled_factory( model, custom_llm_provider, _, _ = litellm.get_llm_provider( model=model, custom_llm_provider=custom_llm_provider ) - model_info = _get_model_info_helper( - model=model, custom_llm_provider=custom_llm_provider - ) + model_info = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) val = model_info.get(key) if val is False: return True @@ -2701,30 +2416,20 @@ def _is_explicitly_disabled_factory( def supports_audio_input(model: str, custom_llm_provider: Optional[str] = None) -> bool: """Check if a given model supports audio input in a chat completion call""" - return _supports_factory( - model=model, custom_llm_provider=custom_llm_provider, key="supports_audio_input" - ) + return _supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_audio_input") def supports_pdf_input(model: str, custom_llm_provider: Optional[str] = None) -> bool: """Check if a given model supports pdf input in a chat completion call""" - return _supports_factory( - model=model, custom_llm_provider=custom_llm_provider, key="supports_pdf_input" - ) + return _supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_pdf_input") -def supports_audio_output( - model: str, custom_llm_provider: Optional[str] = None -) -> bool: +def supports_audio_output(model: str, custom_llm_provider: Optional[str] = None) -> bool: """Check if a given model supports audio output in a chat completion call""" - return _supports_factory( - model=model, custom_llm_provider=custom_llm_provider, key="supports_audio_input" - ) + return _supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_audio_input") -def supports_prompt_caching( - model: str, custom_llm_provider: Optional[str] = None -) -> bool: +def supports_prompt_caching(model: str, custom_llm_provider: Optional[str] = None) -> bool: """ Check if the given model supports prompt caching and return a boolean value. @@ -2745,9 +2450,7 @@ def supports_prompt_caching( ) -def supports_computer_use( - model: str, custom_llm_provider: Optional[str] = None -) -> bool: +def supports_computer_use(model: str, custom_llm_provider: Optional[str] = None) -> bool: """ Check if the given model supports computer use and return a boolean value. @@ -2790,14 +2493,10 @@ def supports_reasoning(model: str, custom_llm_provider: Optional[str] = None) -> """ Check if the given model supports reasoning and return a boolean value. """ - return _supports_factory( - model=model, custom_llm_provider=custom_llm_provider, key="supports_reasoning" - ) + return _supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_reasoning") -def supports_native_structured_output( - model: str, custom_llm_provider: Optional[str] = None -) -> bool: +def supports_native_structured_output(model: str, custom_llm_provider: Optional[str] = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. """ @@ -2808,9 +2507,7 @@ def supports_native_structured_output( ) -def get_supported_regions( - model: str, custom_llm_provider: Optional[str] = None -) -> Optional[List[str]]: +def get_supported_regions(model: str, custom_llm_provider: Optional[str] = None) -> Optional[List[str]]: """ Get a list of supported regions for a given model and provider. @@ -2823,9 +2520,7 @@ def get_supported_regions( model=model, custom_llm_provider=custom_llm_provider ) - model_info = _get_model_info_helper( - model=model, custom_llm_provider=custom_llm_provider - ) + model_info = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) # Get the key used in model_cost to look up supported_regions # since ModelInfoBase doesn't include this field @@ -2852,9 +2547,7 @@ def get_supported_regions( return None -def supports_embedding_image_input( - model: str, custom_llm_provider: Optional[str] = None -) -> bool: +def supports_embedding_image_input(model: str, custom_llm_provider: Optional[str] = None) -> bool: """ Check if the given model supports embedding image input and return a boolean value. """ @@ -2921,9 +2614,7 @@ _CACHE_PRICING_FIELDS = ( ) -def _resolve_builtin_model_cost_entry( - key: str, provider: str -) -> Optional[Dict[str, Any]]: +def _resolve_builtin_model_cost_entry(key: str, provider: str) -> Optional[Dict[str, Any]]: """Best-effort lookup of a built-in ``model_cost`` entry for a custom key whose shape ``get_model_info`` cannot resolve (double provider prefixes like ``bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region aliases). @@ -3002,15 +2693,10 @@ def register_model(model_cost: Union[str, dict]): except Exception: existing_model = {} model_cost_key = key - builtin_entry = _resolve_builtin_model_cost_entry( - key=_key_str, provider=provider - ) + builtin_entry = _resolve_builtin_model_cost_entry(key=_key_str, provider=provider) if builtin_entry is not None: for field in _CACHE_PRICING_FIELDS: - if ( - value.get(field) is None - and builtin_entry.get(field) is not None - ): + if value.get(field) is None and builtin_entry.get(field) is not None: existing_model[field] = builtin_entry[field] elif ( value.get("cache_creation_input_token_cost") is None @@ -3051,9 +2737,7 @@ def register_model(model_cost: Union[str, dict]): # Invalidate case-insensitive lookup map since model_cost was modified _invalidate_model_cost_lowercase_map() - verbose_logger.debug( - f"added/updated model={model_cost_key} in litellm.model_cost: {model_cost_key}" - ) + verbose_logger.debug(f"added/updated model={model_cost_key} in litellm.model_cost: {model_cost_key}") # add new model names to provider lists if value.get("litellm_provider") == "openai": if key not in litellm.open_ai_chat_completion_models: @@ -3105,26 +2789,19 @@ def register_model(model_cost: Union[str, dict]): def _should_drop_param(k, additional_drop_params) -> bool: - if ( - additional_drop_params is not None - and isinstance(additional_drop_params, list) - and k in additional_drop_params - ): + if additional_drop_params is not None and isinstance(additional_drop_params, list) and k in additional_drop_params: return True # allow user to drop specific params for a model - e.g. vllm - logit bias return False -def _get_non_default_params( - passed_params: dict, default_params: dict, additional_drop_params: Optional[list] -) -> dict: +def _get_non_default_params(passed_params: dict, default_params: dict, additional_drop_params: Optional[list]) -> dict: non_default_params = {} for k, v in passed_params.items(): if ( k in default_params and v != default_params[k] - and _should_drop_param(k=k, additional_drop_params=additional_drop_params) - is False + and _should_drop_param(k=k, additional_drop_params=additional_drop_params) is False ): non_default_params[k] = v @@ -3162,11 +2839,7 @@ def get_optional_params_transcription( "timestamp_granularities": None, } - non_default_params = { - k: v - for k, v in passed_params.items() - if (k in default_params and v != default_params[k]) - } + non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])} optional_params = {} ## raise exception if non-default value passed for non-openai/azure embedding calls @@ -3175,9 +2848,8 @@ def get_optional_params_transcription( keys = list(non_default_params.keys()) for k in keys: if ( - (drop_params is True or litellm.drop_params is True) - and k not in supported_params - ): # drop the unsupported non-default values + drop_params is True or litellm.drop_params is True + ) and k not in supported_params: # drop the unsupported non-default values non_default_params.pop(k, None) elif k not in supported_params: raise UnsupportedParamsError( @@ -3239,9 +2911,7 @@ def _map_openai_size_to_vertex_ai_aspect_ratio(size: Optional[str]) -> str: "1792x1024": "16:9", # Landscape "1024x1792": "9:16", # Portrait } - return size_to_aspect_ratio.get( - size, "1:1" - ) # Default to square if size not recognized + return size_to_aspect_ratio.get(size, "1:1") # Default to square if size not recognized def get_optional_params_image_gen( @@ -3275,9 +2945,7 @@ def get_optional_params_image_gen( elif k == "hf_model_name" and custom_llm_provider != "sagemaker": continue elif ( - k.startswith("vertex_") - and custom_llm_provider != "vertex_ai" - and custom_llm_provider != "vertex_ai_beta" + k.startswith("vertex_") and custom_llm_provider != "vertex_ai" and custom_llm_provider != "vertex_ai_beta" ): # allow dynamically setting vertex ai init logic continue passed_params[k] = v @@ -3307,9 +2975,8 @@ def get_optional_params_image_gen( keys = list(non_default_params.keys()) for k in keys: if ( - (litellm.drop_params is True or drop_params is True) - and k not in supported_params - ): # drop the unsupported non-default values + litellm.drop_params is True or drop_params is True + ) and k not in supported_params: # drop the unsupported non-default values non_default_params.pop(k, None) passed_params.pop(k, None) elif k not in supported_params: @@ -3320,9 +2987,7 @@ def get_optional_params_image_gen( return non_default_params if provider_config is not None: - supported_params = provider_config.get_supported_openai_params( - model=model or "" - ) + supported_params = provider_config.get_supported_openai_params(model=model or "") _check_valid_arg(supported_params=supported_params) optional_params = provider_config.map_openai_params( non_default_params=non_default_params, @@ -3340,9 +3005,7 @@ def get_optional_params_image_gen( config_class = litellm.BedrockImageGeneration.get_config_class(model=model) supported_params = config_class.get_supported_openai_params(model=model) _check_valid_arg(supported_params=supported_params) - optional_params = config_class.map_openai_params( - non_default_params=non_default_params, optional_params={} - ) + optional_params = config_class.map_openai_params(non_default_params=non_default_params, optional_params={}) elif custom_llm_provider == "vertex_ai": supported_params = ["n", "size"] """ @@ -3354,15 +3017,11 @@ def get_optional_params_image_gen( # Map OpenAI size parameter to Vertex AI aspectRatio if size is not None: - optional_params["aspectRatio"] = _map_openai_size_to_vertex_ai_aspect_ratio( - size - ) + optional_params["aspectRatio"] = _map_openai_size_to_vertex_ai_aspect_ratio(size) openai_params: list[str] = list(default_params.keys()) if provider_config is not None: - supported_params = provider_config.get_supported_openai_params( - model=model or "" - ) + supported_params = provider_config.get_supported_openai_params(model=model or "") openai_params = list(supported_params) optional_params = add_provider_specific_params_to_optional_params( @@ -3374,9 +3033,7 @@ def get_optional_params_image_gen( ) # remove keys with None or empty dict/list values to avoid sending empty payloads optional_params = { - k: v - for k, v in optional_params.items() - if v is not None and (not isinstance(v, (dict, list)) or len(v) > 0) + k: v for k, v in optional_params.items() if v is not None and (not isinstance(v, (dict, list)) or len(v) > 0) } return optional_params @@ -3394,9 +3051,7 @@ def get_optional_params_embeddings( **kwargs, ): # Lazy load get_supported_openai_params - get_supported_openai_params = getattr( - sys.modules[__name__], "get_supported_openai_params" - ) + get_supported_openai_params = getattr(sys.modules[__name__], "get_supported_openai_params") # retrieve all parameters passed to the function passed_params = locals() @@ -3417,9 +3072,7 @@ def get_optional_params_embeddings( if k not in supported_params: unsupported_params[k] = non_default_params[k] if unsupported_params: - if litellm.drop_params is True or ( - drop_params is not None and drop_params is True - ): + if litellm.drop_params is True or (drop_params is not None and drop_params is True): pass else: raise UnsupportedParamsError( @@ -3427,32 +3080,25 @@ def get_optional_params_embeddings( message=f"{custom_llm_provider} does not support parameters: {unsupported_params}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n", ) - non_default_params = ( - PreProcessNonDefaultParams.embedding_pre_process_non_default_params( - passed_params=passed_params, - special_params=special_params, - custom_llm_provider=custom_llm_provider, - additional_drop_params=additional_drop_params, - model=model, - ) + non_default_params = PreProcessNonDefaultParams.embedding_pre_process_non_default_params( + passed_params=passed_params, + special_params=special_params, + custom_llm_provider=custom_llm_provider, + additional_drop_params=additional_drop_params, + model=model, ) provider_config: Optional[BaseEmbeddingConfig] = None optional_params = {} - if ( - custom_llm_provider is not None - and custom_llm_provider in LlmProviders._member_map_.values() - ): + if custom_llm_provider is not None and custom_llm_provider in LlmProviders._member_map_.values(): provider_config = ProviderConfigManager.get_provider_embedding_config( model=model, provider=LlmProviders(custom_llm_provider), ) if provider_config is not None: - supported_params: Optional[list] = provider_config.get_supported_openai_params( - model=model - ) + supported_params: Optional[list] = provider_config.get_supported_openai_params(model=model) _check_valid_arg(supported_params=supported_params) optional_params = provider_config.map_openai_params( non_default_params=non_default_params, @@ -3469,11 +3115,7 @@ def get_optional_params_embeddings( for param in supported_params: if param in OPENAI_EMBEDDING_PARAMS: continue - if ( - param in passed_params - and passed_params[param] is not None - and param not in optional_params - ): + if param in passed_params and passed_params[param] is not None and param not in optional_params: optional_params[param] = passed_params[param] ## raise exception if non-default value passed for non-openai/azure embedding calls elif custom_llm_provider == "openai": @@ -3543,9 +3185,7 @@ def get_optional_params_embeddings( non_default_params=non_default_params, optional_params={}, kwargs=kwargs ) elif custom_llm_provider == "lm_studio": - supported_params = ( - litellm.LmStudioEmbeddingConfig().get_supported_openai_params() - ) + supported_params = litellm.LmStudioEmbeddingConfig().get_supported_openai_params() _check_valid_arg(supported_params=supported_params) optional_params = litellm.LmStudioEmbeddingConfig().map_openai_params( non_default_params=non_default_params, optional_params={} @@ -3572,9 +3212,7 @@ def get_optional_params_embeddings( supported_params = object.get_supported_openai_params() _check_valid_arg(supported_params=supported_params) - optional_params = object.map_openai_params( - non_default_params=non_default_params, optional_params={} - ) + optional_params = object.map_openai_params(non_default_params=non_default_params, optional_params={}) elif custom_llm_provider == "mistral": supported_params = get_supported_openai_params( model=model, @@ -3606,22 +3244,18 @@ def get_optional_params_embeddings( ) _check_valid_arg(supported_params=supported_params) if litellm.VoyageContextualEmbeddingConfig.is_contextualized_embeddings(model): - optional_params = ( - litellm.VoyageContextualEmbeddingConfig().map_openai_params( - non_default_params=non_default_params, - optional_params={}, - model=model, - drop_params=drop_params if drop_params is not None else False, - ) + optional_params = litellm.VoyageContextualEmbeddingConfig().map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model=model, + drop_params=drop_params if drop_params is not None else False, ) elif litellm.VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings(model): - optional_params = ( - litellm.VoyageMultimodalEmbeddingConfig().map_openai_params( - non_default_params=non_default_params, - optional_params={}, - model=model, - drop_params=drop_params if drop_params is not None else False, - ) + optional_params = litellm.VoyageMultimodalEmbeddingConfig().map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model=model, + drop_params=drop_params if drop_params is not None else False, ) else: optional_params = litellm.VoyageEmbeddingConfig().map_openai_params( @@ -3703,9 +3337,7 @@ def get_optional_params_embeddings( if "dimensions" in non_default_params: optional_params["dimensions"] = non_default_params.pop("dimensions") if len(non_default_params.keys()) > 0: - if ( - litellm.drop_params is True or drop_params is True - ): # drop the unsupported non-default values + if litellm.drop_params is True or drop_params is True: # drop the unsupported non-default values keys = list(non_default_params.keys()) for k in keys: non_default_params.pop(k, None) @@ -3720,9 +3352,7 @@ def get_optional_params_embeddings( and custom_llm_provider not in litellm.openai_compatible_providers ): if len(non_default_params.keys()) > 0: - if ( - litellm.drop_params is True or drop_params is True - ): # drop the unsupported non-default values + if litellm.drop_params is True or drop_params is True: # drop the unsupported non-default values keys = list(non_default_params.keys()) for k in keys: non_default_params.pop(k, None) @@ -3827,9 +3457,7 @@ def _remove_json_schema_refs(schema, max_depth=10): return schema -def _remove_unsupported_params( - non_default_params: dict, supported_openai_params: Optional[List[str]] -) -> dict: +def _remove_unsupported_params(non_default_params: dict, supported_openai_params: Optional[List[str]]) -> dict: """ Remove unsupported params from non_default_params """ @@ -3863,9 +3491,7 @@ def filter_out_litellm_params(kwargs: dict) -> dict: >>> # filtered = {"query": "test"} """ - return { - key: value for key, value in kwargs.items() if key not in all_litellm_params - } + return {key: value for key, value in kwargs.items() if key not in all_litellm_params} class PreProcessNonDefaultParams: @@ -3884,8 +3510,7 @@ class PreProcessNonDefaultParams: # bedrock-mantle configs), never as a request body field continue if k.startswith("aws_") and ( - custom_llm_provider != "bedrock" - and not custom_llm_provider.startswith("sagemaker") + custom_llm_provider != "bedrock" and not custom_llm_provider.startswith("sagemaker") ): # allow dynamically setting boto3 init logic continue elif k == "hf_model_name" and custom_llm_provider != "sagemaker": @@ -3912,10 +3537,7 @@ class PreProcessNonDefaultParams: and k not in additional_endpoint_specific_params and k in default_param_values and v != default_param_values[k] - and _should_drop_param( - k=k, additional_drop_params=additional_drop_params - ) - is False + and _should_drop_param(k=k, additional_drop_params=additional_drop_params) is False ) } @@ -3931,15 +3553,13 @@ class PreProcessNonDefaultParams: remove_sensitive_keys: bool = False, add_provider_specific_params: bool = False, ) -> dict: - non_default_params = ( - PreProcessNonDefaultParams.base_pre_process_non_default_params( - passed_params=passed_params, - special_params=special_params, - custom_llm_provider=custom_llm_provider, - additional_drop_params=additional_drop_params, - default_param_values={k: None for k in OPENAI_EMBEDDING_PARAMS}, - additional_endpoint_specific_params=["input"], - ) + non_default_params = PreProcessNonDefaultParams.base_pre_process_non_default_params( + passed_params=passed_params, + special_params=special_params, + custom_llm_provider=custom_llm_provider, + additional_drop_params=additional_drop_params, + default_param_values={k: None for k in OPENAI_EMBEDDING_PARAMS}, + additional_endpoint_specific_params=["input"], ) return non_default_params @@ -3971,10 +3591,8 @@ def pre_process_non_default_params( if "response_format" in non_default_params: if provider_config is not None: - non_default_params["response_format"] = ( - provider_config.get_json_schema_from_pydantic_object( - response_format=non_default_params["response_format"] - ) + non_default_params["response_format"] = provider_config.get_json_schema_from_pydantic_object( + response_format=non_default_params["response_format"] ) else: non_default_params["response_format"] = type_to_response_format_param( @@ -3990,10 +3608,7 @@ def pre_process_non_default_params( parameters = tool_function.get("parameters", None) if parameters is not None: new_parameters = copy.deepcopy(parameters) - if ( - "additionalProperties" in new_parameters - and new_parameters["additionalProperties"] is False - ): + if "additionalProperties" in new_parameters and new_parameters["additionalProperties"] is False: new_parameters.pop("additionalProperties", None) tool_function["parameters"] = new_parameters @@ -4025,9 +3640,7 @@ def remove_sensitive_keys_from_dict(d: dict) -> dict: return d -def pre_process_optional_params( - passed_params: dict, non_default_params: dict, custom_llm_provider: str -) -> dict: +def pre_process_optional_params(passed_params: dict, non_default_params: dict, custom_llm_provider: str) -> dict: """For .completion(), preprocess optional params""" optional_params: Dict = {} @@ -4042,15 +3655,10 @@ def pre_process_optional_params( non_default_params=passed_params, optional_params=optional_params ) elif custom_llm_provider == "bedrock": - optional_params = ( - litellm.AmazonBedrockGlobalConfig().map_special_auth_params( - non_default_params=passed_params, optional_params=optional_params - ) + optional_params = litellm.AmazonBedrockGlobalConfig().map_special_auth_params( + non_default_params=passed_params, optional_params=optional_params ) - elif ( - custom_llm_provider == "vertex_ai" - or custom_llm_provider == "vertex_ai_beta" - ): + elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta": optional_params = litellm.VertexAIConfig().map_special_auth_params( non_default_params=passed_params, optional_params=optional_params ) @@ -4060,11 +3668,7 @@ def pre_process_optional_params( ) ## raise exception if function calling passed in for a provider that doesn't support it - if ( - "functions" in non_default_params - or "function_call" in non_default_params - or "tools" in non_default_params - ): + if "functions" in non_default_params or "function_call" in non_default_params or "tools" in non_default_params: if ( custom_llm_provider == "ollama" and custom_llm_provider != "text-completion-openai" @@ -4095,23 +3699,13 @@ def pre_process_optional_params( if custom_llm_provider == "ollama": # ollama actually supports json output optional_params["format"] = "json" - litellm.add_function_to_prompt = ( - True # so that main.py adds the function call to the prompt - ) + litellm.add_function_to_prompt = True # so that main.py adds the function call to the prompt if "tools" in non_default_params: - optional_params["functions_unsupported_model"] = ( - non_default_params.pop("tools") - ) - non_default_params.pop( - "tool_choice", None - ) # causes ollama requests to hang + optional_params["functions_unsupported_model"] = non_default_params.pop("tools") + non_default_params.pop("tool_choice", None) # causes ollama requests to hang elif "functions" in non_default_params: - optional_params["functions_unsupported_model"] = ( - non_default_params.pop("functions") - ) - elif ( - litellm.add_function_to_prompt - ): # if user opts to add it to prompt instead + optional_params["functions_unsupported_model"] = non_default_params.pop("functions") + elif litellm.add_function_to_prompt: # if user opts to add it to prompt instead optional_params["functions_unsupported_model"] = non_default_params.pop( "tools", non_default_params.pop("functions", None) ) @@ -4175,9 +3769,7 @@ def get_optional_params( # OpenAI param. passed_params.pop("base_model", None) provider_config: Optional[BaseConfig] = None - if custom_llm_provider is not None and custom_llm_provider in [ - provider.value for provider in LlmProviders - ]: + if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]: provider_config = ProviderConfigManager.get_provider_chat_config( model=model, provider=LlmProviders(custom_llm_provider), @@ -4204,15 +3796,9 @@ def get_optional_params( Args: supported_params: List[str] - supported params from the litellm config """ - verbose_logger.info( - f"\nLiteLLM completion() model= {model}; provider = {custom_llm_provider}" - ) - verbose_logger.debug( - f"\nLiteLLM: Params passed to completion() {passed_params}" - ) - verbose_logger.debug( - f"\nLiteLLM: Non-Default params passed to completion() {non_default_params}" - ) + verbose_logger.info(f"\nLiteLLM completion() model= {model}; provider = {custom_llm_provider}") + verbose_logger.debug(f"\nLiteLLM: Params passed to completion() {passed_params}") + verbose_logger.debug(f"\nLiteLLM: Non-Default params passed to completion() {non_default_params}") unsupported_params = {} for k in non_default_params.keys(): if k not in supported_params: @@ -4229,9 +3815,7 @@ def get_optional_params( unsupported_params[k] = non_default_params[k] if unsupported_params: - if litellm.drop_params is True or ( - drop_params is not None and drop_params is True - ): + if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params.keys(): non_default_params.pop(k, None) else: @@ -4240,16 +3824,12 @@ def get_optional_params( message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.", ) - get_supported_openai_params = getattr( - sys.modules[__name__], "get_supported_openai_params" - ) + get_supported_openai_params = getattr(sys.modules[__name__], "get_supported_openai_params") supported_params = get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider, base_model=base_model ) if supported_params is None: - supported_params = get_supported_openai_params( - model=model, custom_llm_provider="openai" - ) + supported_params = get_supported_openai_params(model=model, custom_llm_provider="openai") supported_params = supported_params or [] allowed_openai_params = allowed_openai_params or [] @@ -4265,32 +3845,20 @@ def get_optional_params( model=model, non_default_params=non_default_params, optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "anthropic_text": optional_params = litellm.AnthropicTextConfig().map_openai_params( model=model, non_default_params=non_default_params, optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) optional_params = litellm.AnthropicTextConfig().map_openai_params( model=model, non_default_params=non_default_params, optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere": @@ -4299,11 +3867,7 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "triton": optional_params = litellm.TritonConfig().map_openai_params( @@ -4318,55 +3882,35 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "replicate": optional_params = litellm.ReplicateConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "predibase": optional_params = litellm.PredibaseConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "huggingface": optional_params = litellm.HuggingFaceChatConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "together_ai": optional_params = litellm.TogetherAIConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "vertex_ai" and ( model in litellm.vertex_chat_models @@ -4380,11 +3924,7 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "gemini": @@ -4392,96 +3932,58 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) - elif custom_llm_provider == "vertex_ai_beta" or ( - custom_llm_provider == "vertex_ai" and "gemini" in model - ): + elif custom_llm_provider == "vertex_ai_beta" or (custom_llm_provider == "vertex_ai" and "gemini" in model): optional_params = litellm.VertexGeminiConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) - elif litellm.VertexAIAnthropicConfig.is_supported_model( - model=model, custom_llm_provider=custom_llm_provider - ): + elif litellm.VertexAIAnthropicConfig.is_supported_model(model=model, custom_llm_provider=custom_llm_provider): optional_params = litellm.VertexAIAnthropicConfig().map_openai_params( model=model, non_default_params=non_default_params, optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "vertex_ai": if model in litellm.vertex_mistral_models: if "codestral" in model: - optional_params = ( - litellm.CodestralTextCompletionConfig().map_openai_params( - model=model, - non_default_params=non_default_params, - optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), - ) + optional_params = litellm.CodestralTextCompletionConfig().map_openai_params( + model=model, + non_default_params=non_default_params, + optional_params=optional_params, + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) else: optional_params = litellm.MistralConfig().map_openai_params( model=model, non_default_params=non_default_params, optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif model in litellm.vertex_ai_ai21_models: optional_params = litellm.VertexAIAi21Config().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif provider_config is not None: optional_params = provider_config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) else: # use generic openai-like param mapping optional_params = litellm.VertexAILlama3Config().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "sagemaker": @@ -4490,11 +3992,7 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "bedrock": BedrockModelInfo = getattr(sys.modules[__name__], "BedrockModelInfo") @@ -4505,61 +4003,36 @@ def get_optional_params( model=model, non_default_params=non_default_params, optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif bedrock_route == "openai": optional_params = litellm.AmazonBedrockOpenAIConfig().map_openai_params( model=model, non_default_params=non_default_params, optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif "anthropic" in bedrock_base_model and bedrock_route == "invoke": - if ( - bedrock_base_model - in litellm.AmazonAnthropicConfig.get_legacy_anthropic_model_names() - ): + if bedrock_base_model in litellm.AmazonAnthropicConfig.get_legacy_anthropic_model_names(): optional_params = litellm.AmazonAnthropicConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) else: - optional_params = ( - litellm.AmazonAnthropicClaudeConfig().map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), - ) + optional_params = litellm.AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif provider_config is not None: optional_params = provider_config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) if bedrock_route == "claude_platform": optional_params = BedrockModelInfo.map_claude_platform_auth_params( @@ -4570,44 +4043,28 @@ def get_optional_params( model=model, non_default_params=non_default_params, optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "ollama": optional_params = litellm.OllamaConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "ollama_chat": optional_params = litellm.OllamaChatConfig().map_openai_params( model=model, non_default_params=non_default_params, optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "nlp_cloud": optional_params = litellm.NLPCloudConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "petals": @@ -4615,55 +4072,35 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "deepinfra": optional_params = litellm.DeepInfraConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "perplexity" and provider_config is not None: optional_params = provider_config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "mistral" or custom_llm_provider == "codestral": optional_params = litellm.MistralConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "text-completion-codestral": optional_params = litellm.CodestralTextCompletionConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "text-completion-inception": @@ -4671,11 +4108,7 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "databricks": @@ -4683,33 +4116,21 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "nvidia_nim": optional_params = litellm.NvidiaNimConfig().map_openai_params( model=model, non_default_params=non_default_params, optional_params=optional_params, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "cerebras": optional_params = litellm.CerebrasConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "xai": optional_params = litellm.XAIChatConfig().map_openai_params( @@ -4722,110 +4143,70 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "fireworks_ai": optional_params = litellm.FireworksAIConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "volcengine": optional_params = litellm.VolcEngineConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "hosted_vllm": optional_params = litellm.HostedVLLMChatConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "vllm": optional_params = litellm.VLLMConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "groq": optional_params = litellm.GroqChatConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "bedrock_mantle": optional_params = litellm.BedrockMantleChatConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "deepseek": optional_params = litellm.DeepSeekChatConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "openrouter": optional_params = litellm.OpenrouterConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "watsonx": optional_params = litellm.IBMWatsonXChatConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) # WatsonX-text param check for param in passed_params.keys(): @@ -4838,61 +4219,37 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "openai": optional_params = litellm.OpenAIConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "nebius": optional_params = litellm.NebiusConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif custom_llm_provider == "azure": _azure_detection_model = base_model or model - if litellm.AzureOpenAIO1Config().is_o_series_model( - model=_azure_detection_model - ): + if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model): optional_params = litellm.AzureOpenAIO1Config().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=_azure_detection_model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) - elif litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model( - model=_azure_detection_model - ): + elif litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model=_azure_detection_model): optional_params = litellm.AzureOpenAIGPT5Config().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=_azure_detection_model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) else: verbose_logger.debug( @@ -4911,33 +4268,21 @@ def get_optional_params( optional_params=optional_params, model=_azure_detection_model, api_version=api_version, # type: ignore - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) elif provider_config is not None: optional_params = provider_config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) else: # assume passing in params for openai-like api optional_params = litellm.OpenAILikeChatConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=( - drop_params - if drop_params is not None and isinstance(drop_params, bool) - else False - ), + drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False), ) # if user passed in non-default kwargs for specific providers/models, pass them along optional_params = add_provider_specific_params_to_optional_params( @@ -4976,18 +4321,9 @@ def add_provider_specific_params_to_optional_params( Add provider specific params to optional_params """ - if ( - custom_llm_provider - in ["openai", "azure", "text-completion-openai"] - + litellm.openai_compatible_providers - ): + if custom_llm_provider in ["openai", "azure", "text-completion-openai"] + litellm.openai_compatible_providers: # for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai-python/blob/ac33853ba10d13ac149b1fa3ca6dba7d613065c9/src/openai/resources/models.py#L46 - if ( - _should_drop_param( - k="extra_body", additional_drop_params=additional_drop_params - ) - is False - ): + if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False: extra_body = dict(passed_params.pop("extra_body", None) or {}) for k in passed_params.keys(): if k not in openai_params and passed_params[k] is not None: @@ -5000,34 +4336,22 @@ def add_provider_specific_params_to_optional_params( } if additional_drop_params is not None: - processed_extra_body = { - k: v - for k, v in initial_extra_body.items() - if k not in additional_drop_params - } + processed_extra_body = {k: v for k, v in initial_extra_body.items() if k not in additional_drop_params} else: processed_extra_body = initial_extra_body - _ensure_extra_body_is_safe = getattr( - sys.modules[__name__], "_ensure_extra_body_is_safe" - ) - optional_params["extra_body"] = _ensure_extra_body_is_safe( - extra_body=processed_extra_body - ) + _ensure_extra_body_is_safe = getattr(sys.modules[__name__], "_ensure_extra_body_is_safe") + optional_params["extra_body"] = _ensure_extra_body_is_safe(extra_body=processed_extra_body) else: for k in passed_params.keys(): if k not in openai_params and passed_params[k] is not None: - if _should_drop_param( - k=k, additional_drop_params=additional_drop_params - ): + if _should_drop_param(k=k, additional_drop_params=additional_drop_params): continue optional_params[k] = passed_params[k] return optional_params -def _apply_openai_param_overrides( - optional_params: dict, non_default_params: dict, allowed_openai_params: list -): +def _apply_openai_param_overrides(optional_params: dict, non_default_params: dict, allowed_openai_params: list): """ If user passes in allowed_openai_params, apply them to optional_params @@ -5122,13 +4446,9 @@ def _get_deployment_order(deployment: Union[Dict, Any]) -> Optional[int]: return order -def _get_order_filtered_deployments( - healthy_deployments: List[Dict], target_order: Optional[int] = None -) -> List: +def _get_order_filtered_deployments(healthy_deployments: List[Dict], target_order: Optional[int] = None) -> List: if target_order is not None: - filtered = [ - d for d in healthy_deployments if _get_deployment_order(d) == target_order - ] + filtered = [d for d in healthy_deployments if _get_deployment_order(d) == target_order] if filtered: return filtered # target_order doesn't match any deployment (e.g., external fallback model) — return all @@ -5136,18 +4456,13 @@ def _get_order_filtered_deployments( # Default: pick min order group _valid_orders: List[int] = [ - o - for deployment in healthy_deployments - for o in [_get_deployment_order(deployment)] - if o is not None + o for deployment in healthy_deployments for o in [_get_deployment_order(deployment)] if o is not None ] min_order: Optional[int] = min(_valid_orders) if _valid_orders else None if min_order is not None: filtered_deployments = [ - deployment - for deployment in healthy_deployments - if _get_deployment_order(deployment) == min_order + deployment for deployment in healthy_deployments if _get_deployment_order(deployment) == min_order ] return filtered_deployments @@ -5174,16 +4489,10 @@ def _get_excluded_filtered_deployments( return healthy_deployments excluded_set = set(excluded_deployment_ids) - return [ - d - for d in healthy_deployments - if (d.get("model_info") or {}).get("id") not in excluded_set - ] + return [d for d in healthy_deployments if (d.get("model_info") or {}).get("id") not in excluded_set] -def _get_model_region( - custom_llm_provider: str, litellm_params: LiteLLM_Params -) -> Optional[str]: +def _get_model_region(custom_llm_provider: str, litellm_params: LiteLLM_Params) -> Optional[str]: """ Return the region for a model, for a given provider """ @@ -5220,14 +4529,10 @@ def _infer_model_region(litellm_params: LiteLLM_Params) -> Optional[AllowedModel model=litellm_params.model, litellm_params=litellm_params ) - model_region = _get_model_region( - custom_llm_provider=custom_llm_provider, litellm_params=litellm_params - ) + model_region = _get_model_region(custom_llm_provider=custom_llm_provider, litellm_params=litellm_params) if model_region is None: - verbose_logger.debug( - "Cannot infer model region for model: {}".format(litellm_params.model) - ) + verbose_logger.debug("Cannot infer model region for model: {}".format(litellm_params.model)) return None if custom_llm_provider == "azure": @@ -5283,9 +4588,7 @@ def _is_region_us(litellm_params: LiteLLM_Params) -> bool: return False -def is_region_allowed( - litellm_params: LiteLLM_Params, allowed_model_region: str -) -> bool: +def is_region_allowed(litellm_params: LiteLLM_Params, allowed_model_region: str) -> bool: """ Return true/false if a deployment is in the EU """ @@ -5294,9 +4597,7 @@ def is_region_allowed( return False -def get_model_region( - litellm_params: LiteLLM_Params, mode: Optional[str] -) -> Optional[str]: +def get_model_region(litellm_params: LiteLLM_Params, mode: Optional[str]) -> Optional[str]: """ Pass the litellm params for an azure model, and get back the region """ @@ -5394,9 +4695,7 @@ def get_api_key(llm_provider: str, dynamic_api_key: Optional[str]): api_key = api_key or litellm.ai21_key or get_secret("AI211_API_KEY") # aleph_alpha elif llm_provider == "aleph_alpha": - api_key = ( - api_key or litellm.aleph_alpha_key or get_secret("ALEPH_ALPHA_API_KEY") - ) + api_key = api_key or litellm.aleph_alpha_key or get_secret("ALEPH_ALPHA_API_KEY") # baseten elif llm_provider == "baseten": api_key = api_key or litellm.baseten_key or get_secret("BASETEN_API_KEY") @@ -5405,9 +4704,7 @@ def get_api_key(llm_provider: str, dynamic_api_key: Optional[str]): api_key = api_key or litellm.cohere_key or get_secret("COHERE_API_KEY") # huggingface elif llm_provider == "huggingface": - api_key = ( - api_key or litellm.huggingface_key or get_secret("HUGGINGFACE_API_KEY") - ) + api_key = api_key or litellm.huggingface_key or get_secret("HUGGINGFACE_API_KEY") # nlp_cloud elif llm_provider == "nlp_cloud": api_key = api_key or litellm.nlp_cloud_key or get_secret("NLP_CLOUD_API_KEY") @@ -5417,10 +4714,7 @@ def get_api_key(llm_provider: str, dynamic_api_key: Optional[str]): # together_ai elif llm_provider == "together_ai": api_key = ( - api_key - or litellm.togetherai_api_key - or get_secret("TOGETHERAI_API_KEY") - or get_secret("TOGETHER_AI_TOKEN") + api_key or litellm.togetherai_api_key or get_secret("TOGETHERAI_API_KEY") or get_secret("TOGETHER_AI_TOKEN") ) # nebius elif llm_provider == "nebius": @@ -5539,9 +4833,7 @@ def _strip_model_name(model: str, custom_llm_provider: Optional[str]) -> str: if custom_llm_provider and custom_llm_provider in ["bedrock", "bedrock_converse"]: stripped_bedrock_model = _get_base_bedrock_model(model_name=model) return stripped_bedrock_model - elif custom_llm_provider and ( - custom_llm_provider == "vertex_ai" or custom_llm_provider == "gemini" - ): + elif custom_llm_provider and (custom_llm_provider == "vertex_ai" or custom_llm_provider == "gemini"): strip_version = _strip_stable_vertex_version(model_name=model) return strip_version elif custom_llm_provider and (custom_llm_provider == "databricks"): @@ -5688,20 +4980,13 @@ def _check_provider_match(model_info: dict, custom_llm_provider: Optional[str]) so normalising the two cases keeps custom pricing applied consistently. """ if custom_llm_provider and ( - model_info.get("litellm_provider") is not None - and model_info["litellm_provider"] != custom_llm_provider + model_info.get("litellm_provider") is not None and model_info["litellm_provider"] != custom_llm_provider ): - if custom_llm_provider == "vertex_ai" and model_info[ - "litellm_provider" - ].startswith("vertex_ai"): + if custom_llm_provider == "vertex_ai" and model_info["litellm_provider"].startswith("vertex_ai"): return True - elif custom_llm_provider == "fireworks_ai" and model_info[ - "litellm_provider" - ].startswith("fireworks_ai"): + elif custom_llm_provider == "fireworks_ai" and model_info["litellm_provider"].startswith("fireworks_ai"): return True - elif custom_llm_provider.startswith("bedrock") and model_info[ - "litellm_provider" - ].startswith("bedrock"): + elif custom_llm_provider.startswith("bedrock") and model_info["litellm_provider"].startswith("bedrock"): return True elif ( custom_llm_provider == "litellm_proxy" @@ -5746,27 +5031,19 @@ def _get_potential_model_names( except Exception: split_model = model combined_model_name = model - stripped_model_name = _strip_model_name( - model=model, custom_llm_provider=custom_llm_provider - ) + stripped_model_name = _strip_model_name(model=model, custom_llm_provider=custom_llm_provider) combined_stripped_model_name = stripped_model_name - elif ( - custom_llm_provider and model.startswith(custom_llm_provider + "/") + elif custom_llm_provider and model.startswith( + custom_llm_provider + "/" ): # handle case where custom_llm_provider is provided and model starts with custom_llm_provider split_model = model.split("/", 1)[1] combined_model_name = model - stripped_model_name = _strip_model_name( - model=split_model, custom_llm_provider=custom_llm_provider - ) - combined_stripped_model_name = "{}/{}".format( - custom_llm_provider, stripped_model_name - ) + stripped_model_name = _strip_model_name(model=split_model, custom_llm_provider=custom_llm_provider) + combined_stripped_model_name = "{}/{}".format(custom_llm_provider, stripped_model_name) else: split_model = model combined_model_name = "{}/{}".format(custom_llm_provider, model) - stripped_model_name = _strip_model_name( - model=model, custom_llm_provider=custom_llm_provider - ) + stripped_model_name = _strip_model_name(model=model, custom_llm_provider=custom_llm_provider) combined_stripped_model_name = "{}/{}".format( custom_llm_provider, stripped_model_name, @@ -5822,9 +5099,7 @@ def _cached_get_model_info_helper( ) -def get_provider_info( - model: str, custom_llm_provider: Optional[str] -) -> Optional[ProviderSpecificModelInfo]: +def get_provider_info(model: str, custom_llm_provider: Optional[str]) -> Optional[ProviderSpecificModelInfo]: ## PROVIDER-SPECIFIC INFORMATION # if custom_llm_provider == "predibase": # _model_info["supports_response_schema"] = True @@ -5877,19 +5152,13 @@ def _get_model_info_helper( elif model + "@latest" in litellm.vertex_ai_ai21_models: model = model + "@latest" ########################## - potential_model_names = _get_potential_model_names( - model=model, custom_llm_provider=custom_llm_provider - ) + potential_model_names = _get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider) - verbose_logger.debug( - f"checking potential_model_names in litellm.model_cost: {potential_model_names}" - ) + verbose_logger.debug(f"checking potential_model_names in litellm.model_cost: {potential_model_names}") combined_model_name = potential_model_names["combined_model_name"] stripped_model_name = potential_model_names["stripped_model_name"] - combined_stripped_model_name = potential_model_names[ - "combined_stripped_model_name" - ] + combined_stripped_model_name = potential_model_names["combined_stripped_model_name"] split_model = potential_model_names["split_model"] custom_llm_provider = potential_model_names["custom_llm_provider"] model_cost_custom_llm_provider = custom_llm_provider @@ -6007,9 +5276,7 @@ def _get_model_info_helper( raise ValueError( "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" ) - _input_cost_per_token: Optional[float] = _model_info.get( - "input_cost_per_token" - ) + _input_cost_per_token: Optional[float] = _model_info.get("input_cost_per_token") if _input_cost_per_token is None: # default value to 0, be noisy about this verbose_logger.debug( @@ -6019,9 +5286,7 @@ def _get_model_info_helper( ) _input_cost_per_token = 0 - _output_cost_per_token: Optional[float] = _model_info.get( - "output_cost_per_token" - ) + _output_cost_per_token: Optional[float] = _model_info.get("output_cost_per_token") if _output_cost_per_token is None: # default value to 0, be noisy about this verbose_logger.debug( @@ -6037,21 +5302,13 @@ def _get_model_info_helper( max_input_tokens=_model_info.get("max_input_tokens", None), max_output_tokens=_model_info.get("max_output_tokens", None), input_cost_per_token=_input_cost_per_token, - input_cost_per_token_flex=_model_info.get( - "input_cost_per_token_flex", None - ), - input_cost_per_token_priority=_model_info.get( - "input_cost_per_token_priority", None - ), - cache_creation_input_token_cost=_model_info.get( - "cache_creation_input_token_cost", None - ), + input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None), + input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None), + cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None), cache_creation_input_token_cost_above_200k_tokens=_model_info.get( "cache_creation_input_token_cost_above_200k_tokens", None ), - cache_read_input_token_cost=_model_info.get( - "cache_read_input_token_cost", None - ), + cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None ), @@ -6067,79 +5324,43 @@ def _get_model_info_helper( cache_read_input_token_cost_above_512k_tokens=_model_info.get( "cache_read_input_token_cost_above_512k_tokens", None ), - cache_read_input_token_cost_flex=_model_info.get( - "cache_read_input_token_cost_flex", None - ), - cache_read_input_token_cost_priority=_model_info.get( - "cache_read_input_token_cost_priority", None - ), + cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None), + cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None), cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), - input_cost_per_character=_model_info.get( - "input_cost_per_character", None - ), - input_cost_per_token_above_128k_tokens=_model_info.get( - "input_cost_per_token_above_128k_tokens", None - ), - input_cost_per_token_above_200k_tokens=_model_info.get( - "input_cost_per_token_above_200k_tokens", None - ), + input_cost_per_character=_model_info.get("input_cost_per_character", None), + input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None), + input_cost_per_token_above_200k_tokens=_model_info.get("input_cost_per_token_above_200k_tokens", None), input_cost_per_token_above_200k_tokens_priority=_model_info.get( "input_cost_per_token_above_200k_tokens_priority", None ), - input_cost_per_token_above_272k_tokens=_model_info.get( - "input_cost_per_token_above_272k_tokens", None - ), + input_cost_per_token_above_272k_tokens=_model_info.get("input_cost_per_token_above_272k_tokens", None), input_cost_per_token_above_272k_tokens_priority=_model_info.get( "input_cost_per_token_above_272k_tokens_priority", None ), - input_cost_per_token_above_512k_tokens=_model_info.get( - "input_cost_per_token_above_512k_tokens", None - ), + input_cost_per_token_above_512k_tokens=_model_info.get("input_cost_per_token_above_512k_tokens", None), input_cost_per_query=_model_info.get("input_cost_per_query", None), input_cost_per_second=_model_info.get("input_cost_per_second", None), - input_cost_per_audio_token=_model_info.get( - "input_cost_per_audio_token", None - ), - input_cost_per_image_token=_model_info.get( - "input_cost_per_image_token", None - ), + input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None), + input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None), input_cost_per_image=_model_info.get("input_cost_per_image", None), - input_cost_per_audio_per_second=_model_info.get( - "input_cost_per_audio_per_second", None - ), - input_cost_per_video_per_second=_model_info.get( - "input_cost_per_video_per_second", None - ), - input_cost_per_token_batches=_model_info.get( - "input_cost_per_token_batches" - ), - output_cost_per_token_batches=_model_info.get( - "output_cost_per_token_batches" - ), + input_cost_per_audio_per_second=_model_info.get("input_cost_per_audio_per_second", None), + input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None), + input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"), + output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"), output_cost_per_token=_output_cost_per_token, - output_cost_per_token_flex=_model_info.get( - "output_cost_per_token_flex", None - ), - output_cost_per_token_priority=_model_info.get( - "output_cost_per_token_priority", None - ), + output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None), + output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None), regional_processing_uplift_multiplier_eu=_model_info.get( "regional_processing_uplift_multiplier_eu", None ), regional_processing_uplift_multiplier_us=_model_info.get( "regional_processing_uplift_multiplier_us", None ), - output_cost_per_audio_token=_model_info.get( - "output_cost_per_audio_token", None - ), - output_cost_per_character=_model_info.get( - "output_cost_per_character", None - ), - output_cost_per_reasoning_token=_model_info.get( - "output_cost_per_reasoning_token", None - ), + output_cost_per_audio_token=_model_info.get("output_cost_per_audio_token", None), + output_cost_per_character=_model_info.get("output_cost_per_character", None), + output_cost_per_reasoning_token=_model_info.get("output_cost_per_reasoning_token", None), output_cost_per_token_above_128k_tokens=_model_info.get( "output_cost_per_token_above_128k_tokens", None ), @@ -6162,89 +5383,45 @@ def _get_model_info_helper( "output_cost_per_token_above_512k_tokens", None ), output_cost_per_second=_model_info.get("output_cost_per_second", None), - output_cost_per_second_1080p=_model_info.get( - "output_cost_per_second_1080p", None - ), - output_cost_per_video_per_second=_model_info.get( - "output_cost_per_video_per_second", None - ), + output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None), + output_cost_per_video_per_second=_model_info.get("output_cost_per_video_per_second", None), output_cost_per_image=_model_info.get("output_cost_per_image", None), - output_cost_per_image_token=_model_info.get( - "output_cost_per_image_token", None - ), + output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None), output_vector_size=_model_info.get("output_vector_size", None), - citation_cost_per_token=_model_info.get( - "citation_cost_per_token", None - ), + citation_cost_per_token=_model_info.get("citation_cost_per_token", None), tiered_pricing=_model_info.get("tiered_pricing", None), - litellm_provider=_model_info.get( - "litellm_provider", custom_llm_provider - ), + litellm_provider=_model_info.get("litellm_provider", custom_llm_provider), mode=_model_info.get("mode"), # type: ignore - supports_system_messages=_model_info.get( - "supports_system_messages", None - ), - supports_response_schema=_model_info.get( - "supports_response_schema", None - ), + supports_system_messages=_model_info.get("supports_system_messages", None), + supports_response_schema=_model_info.get("supports_response_schema", None), supports_vision=_model_info.get("supports_vision", None), - supports_function_calling=_model_info.get( - "supports_function_calling", None - ), + supports_function_calling=_model_info.get("supports_function_calling", None), supports_tool_choice=_model_info.get("supports_tool_choice", None), - supports_assistant_prefill=_model_info.get( - "supports_assistant_prefill", None - ), - supports_prompt_caching=_model_info.get( - "supports_prompt_caching", None - ), + supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None), + supports_prompt_caching=_model_info.get("supports_prompt_caching", None), supports_audio_input=_model_info.get("supports_audio_input", None), supports_audio_output=_model_info.get("supports_audio_output", None), supports_pdf_input=_model_info.get("supports_pdf_input", None), - supports_embedding_image_input=_model_info.get( - "supports_embedding_image_input", None - ), - supports_native_streaming=_model_info.get( - "supports_native_streaming", None - ), - supports_native_structured_output=_model_info.get( - "supports_native_structured_output", None - ), + supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None), + supports_native_streaming=_model_info.get("supports_native_streaming", None), + supports_native_structured_output=_model_info.get("supports_native_structured_output", None), supports_web_search=_model_info.get("supports_web_search", None), supports_url_context=_model_info.get("supports_url_context", None), supports_reasoning=_model_info.get("supports_reasoning", None), - supports_none_reasoning_effort=_model_info.get( - "supports_none_reasoning_effort", None - ), - supports_minimal_reasoning_effort=_model_info.get( - "supports_minimal_reasoning_effort", None - ), - supports_low_reasoning_effort=_model_info.get( - "supports_low_reasoning_effort", None - ), - supports_xhigh_reasoning_effort=_model_info.get( - "supports_xhigh_reasoning_effort", None - ), - supports_max_reasoning_effort=_model_info.get( - "supports_max_reasoning_effort", None - ), - bedrock_output_config_effort_ceiling=_model_info.get( - "bedrock_output_config_effort_ceiling", None - ), + supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None), + supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None), + supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None), + supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None), + supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None), + bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None), supports_computer_use=_model_info.get("supports_computer_use", None), - search_context_cost_per_query=_model_info.get( - "search_context_cost_per_query", None - ), + search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None), tpm=_model_info.get("tpm", None), rpm=_model_info.get("rpm", None), ocr_cost_per_page=_model_info.get("ocr_cost_per_page", None), ocr_cost_per_credit=_model_info.get("ocr_cost_per_credit", None), - annotation_cost_per_page=_model_info.get( - "annotation_cost_per_page", None - ), - provider_specific_entry=_model_info.get( - "provider_specific_entry", None - ), + annotation_cost_per_page=_model_info.get("annotation_cost_per_page", None), + provider_specific_entry=_model_info.get("provider_specific_entry", None), uses_embed_content=_model_info.get("uses_embed_content", None), supports_image_size=_model_info.get("supports_image_size", None), ) @@ -6263,9 +5440,7 @@ def _build_model_info( api_base: Optional[str] = None, api_key: Optional[str] = None, ) -> ModelInfo: - supported_openai_params = litellm.get_supported_openai_params( - model=model, custom_llm_provider=custom_llm_provider - ) + supported_openai_params = litellm.get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider) _model_info = _get_model_info_helper( model=model, @@ -6274,9 +5449,7 @@ def _build_model_info( api_key=api_key, ) - provider_info = get_provider_info( - model=model, custom_llm_provider=custom_llm_provider - ) + provider_info = get_provider_info(model=model, custom_llm_provider=custom_llm_provider) if provider_info: for key, value in provider_info.items(): if value is not None: @@ -6294,9 +5467,7 @@ def _cached_get_model_info( custom_llm_provider: Optional[str] = None, api_base: Optional[str] = None, ) -> ModelInfo: - return _build_model_info( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base - ) + return _build_model_info(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base) def get_model_info( @@ -6478,9 +5649,7 @@ def function_to_dict(input_function) -> dict: # noqa: C901 "enum": param_enum, } - parameters[param_name] = dict( - [(k, v) for k, v in param_dict.items() if isinstance(v, str)] - ) + parameters[param_name] = dict([(k, v) for k, v in param_dict.items() if isinstance(v, str)]) # Check if the parameter has no default value (i.e., it's required) if param.default == param.empty: @@ -6569,10 +5738,7 @@ def get_provider_fields(custom_llm_provider: str) -> List[ProviderField]: def create_proxy_transport_and_mounts(): - proxies = { - key: None if url is None else Proxy(url=url) - for key, url in get_environment_proxies().items() - } + proxies = {key: None if url is None else Proxy(url=url) for key, url in get_environment_proxies().items()} sync_proxy_mounts = {} async_proxy_mounts = {} @@ -6636,21 +5802,12 @@ def validate_environment( else: missing_keys.append("OPENAI_API_KEY") elif custom_llm_provider == "azure": - if ( - "AZURE_API_BASE" in os.environ - and "AZURE_API_VERSION" in os.environ - and "AZURE_API_KEY" in os.environ - ): + if "AZURE_API_BASE" in os.environ and "AZURE_API_VERSION" in os.environ and "AZURE_API_KEY" in os.environ: keys_in_environment = True else: - missing_keys.extend( - ["AZURE_API_BASE", "AZURE_API_VERSION", "AZURE_API_KEY"] - ) + missing_keys.extend(["AZURE_API_BASE", "AZURE_API_VERSION", "AZURE_API_KEY"]) elif custom_llm_provider == "anthropic": - if ( - "ANTHROPIC_API_KEY" in os.environ - or "ANTHROPIC_AUTH_TOKEN" in os.environ - ): + if "ANTHROPIC_API_KEY" in os.environ or "ANTHROPIC_AUTH_TOKEN" in os.environ: keys_in_environment = True else: missing_keys.append("ANTHROPIC_API_KEY") @@ -6715,18 +5872,13 @@ def validate_environment( else: missing_keys.append("NLP_CLOUD_API_KEY") elif custom_llm_provider == "bedrock" or custom_llm_provider == "sagemaker": - if ( - "AWS_ACCESS_KEY_ID" in os.environ - and "AWS_SECRET_ACCESS_KEY" in os.environ - ) or ( + if ("AWS_ACCESS_KEY_ID" in os.environ and "AWS_SECRET_ACCESS_KEY" in os.environ) or ( # IAM role, profile, or web identity auth don't require access keys "AWS_ROLE_ARN" in os.environ or "AWS_PROFILE" in os.environ or "AWS_WEB_IDENTITY_TOKEN_FILE" in os.environ - or "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" - in os.environ # ECS task role - or "AWS_CONTAINER_CREDENTIALS_FULL_URI" - in os.environ # ECS/Fargate full URI credential delivery + or "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" in os.environ # ECS task role + or "AWS_CONTAINER_CREDENTIALS_FULL_URI" in os.environ # ECS/Fargate full URI credential delivery ): keys_in_environment = True else: @@ -6793,18 +5945,12 @@ def validate_environment( keys_in_environment = True else: missing_keys.append("VOLCENGINE_API_KEY") - elif ( - custom_llm_provider == "codestral" - or custom_llm_provider == "text-completion-codestral" - ): + elif custom_llm_provider == "codestral" or custom_llm_provider == "text-completion-codestral": if "CODESTRAL_API_KEY" in os.environ: keys_in_environment = True else: missing_keys.append("CODESTRAL_API_KEY") - elif ( - custom_llm_provider == "inception" - or custom_llm_provider == "text-completion-inception" - ): + elif custom_llm_provider == "inception" or custom_llm_provider == "text-completion-inception": if "INCEPTION_API_KEY" in os.environ: keys_in_environment = True else: @@ -6851,8 +5997,7 @@ def validate_environment( missing_keys.append("FIREWORKS_AI_API_KEY") elif custom_llm_provider == "cloudflare": if "CLOUDFLARE_API_KEY" in os.environ and ( - "CLOUDFLARE_ACCOUNT_ID" in os.environ - or "CLOUDFLARE_API_BASE" in os.environ + "CLOUDFLARE_ACCOUNT_ID" in os.environ or "CLOUDFLARE_API_BASE" in os.environ ): keys_in_environment = True else: @@ -6903,10 +6048,7 @@ def validate_environment( missing_keys.append("OPENAI_API_KEY") ## anthropic elif model in litellm.anthropic_models: - if ( - "ANTHROPIC_API_KEY" in os.environ - or "ANTHROPIC_AUTH_TOKEN" in os.environ - ): + if "ANTHROPIC_API_KEY" in os.environ or "ANTHROPIC_AUTH_TOKEN" in os.environ: keys_in_environment = True else: missing_keys.append("ANTHROPIC_API_KEY") @@ -7046,10 +6188,7 @@ def prompt_token_calculator(model, messages): def valid_model(model): try: # for a given model name, check if the user has the right permissions to access the model - if ( - model in litellm.open_ai_chat_completion_models - or model in litellm.open_ai_text_completion_models - ): + if model in litellm.open_ai_chat_completion_models or model in litellm.open_ai_text_completion_models: openai.models.retrieve(model) else: messages = [{"role": "user", "content": "Hello World"}] @@ -7071,9 +6210,7 @@ def check_valid_key(model: str, api_key: str): """ messages = [{"role": "user", "content": "Hey, how's it going?"}] try: - litellm.completion( - model=model, messages=messages, api_key=api_key, max_tokens=10 - ) + litellm.completion(model=model, messages=messages, api_key=api_key, max_tokens=10) return True except AuthenticationError: return False @@ -7265,9 +6402,7 @@ class TextCompletionStreamWrapper: response["created"] = chunk.get("created", None) response["model"] = chunk.get("model", None) text_choices = TextChoices() - if isinstance( - chunk, Choices - ): # chunk should always be of type StreamingChoices + if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices raise Exception delta = chunk["choices"][0]["delta"] text_choices["text"] = delta["content"] @@ -7277,17 +6412,12 @@ class TextCompletionStreamWrapper: response["choices"] = [text_choices] # only pass usage when stream_options["include_usage"] is True - if ( - self.stream_options - and self.stream_options.get("include_usage", False) is True - ): + if self.stream_options and self.stream_options.get("include_usage", False) is True: response["usage"] = chunk.get("usage", None) return response except Exception as e: - raise Exception( - f"Error occurred converting to text completion object - chunk: {chunk}; Error: {str(e)}" - ) + raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {str(e)}") def __next__(self): # model_response = ModelResponse(stream=True, model=self.model) @@ -7323,9 +6453,7 @@ class TextCompletionStreamWrapper: raise StopAsyncIteration -def mock_completion_streaming_obj( - model_response, mock_response, model, n: Optional[int] = None -): +def mock_completion_streaming_obj(model_response, mock_response, model, n: Optional[int] = None): if isinstance(mock_response, litellm.MockException): raise mock_response if isinstance(mock_response, ModelResponseStream): @@ -7340,9 +6468,7 @@ def mock_completion_streaming_obj( for j in range(n): _streaming_choice = litellm.utils.StreamingChoices( index=j, - delta=litellm.utils.Delta( - role="assistant", content=mock_response[i : i + 3] - ), + delta=litellm.utils.Delta(role="assistant", content=mock_response[i : i + 3]), ) _all_choices.append(_streaming_choice) model_response.choices = _all_choices @@ -7369,9 +6495,7 @@ async def async_mock_completion_streaming_obj( for j in range(n): _streaming_choice = litellm.utils.StreamingChoices( index=j, - delta=litellm.utils.Delta( - role="assistant", content=mock_response[i : i + 3] - ), + delta=litellm.utils.Delta(role="assistant", content=mock_response[i : i + 3]), ) _all_choices.append(_streaming_choice) model_response.choices = _all_choices @@ -7401,13 +6525,9 @@ def process_system_message(system_message, max_tokens, model): system_message_tokens = get_token_count([system_message_event], model) if system_message_tokens > max_tokens: - print_verbose( - "`tokentrimmer`: Warning, system message exceeds token limit. Trimming..." - ) + print_verbose("`tokentrimmer`: Warning, system message exceeds token limit. Trimming...") # shorten system message to fit within max_tokens - new_system_message = shorten_message_to_fit_limit( - system_message_event, max_tokens, model - ) + new_system_message = shorten_message_to_fit_limit(system_message_event, max_tokens, model) system_message_tokens = get_token_count([new_system_message], model) return system_message_event, max_tokens - system_message_tokens @@ -7424,9 +6544,7 @@ def process_messages(messages, max_tokens, model): verbose_logger.debug(f"processing final_messages: {final_messages}") used_tokens = get_token_count(final_messages, model) available_tokens = max_tokens - used_tokens - verbose_logger.debug( - f"used_tokens: {used_tokens}, available_tokens: {available_tokens}" - ) + verbose_logger.debug(f"used_tokens: {used_tokens}, available_tokens: {available_tokens}") if available_tokens <= 3: break @@ -7437,21 +6555,15 @@ def process_messages(messages, max_tokens, model): max_tokens=max_tokens, model=model, ) - verbose_logger.debug( - f"final_messages after attempt_message_addition: {final_messages}" - ) + verbose_logger.debug(f"final_messages after attempt_message_addition: {final_messages}") verbose_logger.debug(f"Final messages: {final_messages}") return final_messages -def attempt_message_addition( - final_messages, message, available_tokens, max_tokens, model -): +def attempt_message_addition(final_messages, message, available_tokens, max_tokens, model): temp_messages = [message] + final_messages temp_message_tokens = get_token_count(messages=temp_messages, model=model) - verbose_logger.debug( - f"temp_message_tokens: {temp_message_tokens}, max_tokens: {max_tokens}" - ) + verbose_logger.debug(f"temp_message_tokens: {temp_message_tokens}, max_tokens: {max_tokens}") if temp_message_tokens <= max_tokens: return temp_messages @@ -7461,9 +6573,7 @@ def attempt_message_addition( # fit updated_message to be within temp_message_tokens - max_tokens (aka the amount temp_message_tokens is greate than max_tokens) updated_message = shorten_message_to_fit_limit(message, available_tokens, model) if can_add_message(updated_message, final_messages, max_tokens, model): - verbose_logger.debug( - "can add message, returning [updated_message] + final_messages" - ) + verbose_logger.debug("can add message, returning [updated_message] + final_messages") return [updated_message] + final_messages else: verbose_logger.debug("cannot add message, returning final_messages") @@ -7480,9 +6590,7 @@ def get_token_count(messages, model): return token_counter(model=model, messages=messages) -def shorten_message_to_fit_limit( - message, tokens_needed, model: Optional[str], raise_error_on_max_limit: bool = False -): +def shorten_message_to_fit_limit(message, tokens_needed, model: Optional[str], raise_error_on_max_limit: bool = False): """ Shorten a message to fit within a token limit by removing characters from the middle. @@ -7507,9 +6615,7 @@ def shorten_message_to_fit_limit( while attempts < MAX_TOKEN_TRIMMING_ATTEMPTS: verbose_logger.debug(f"getting token count for message: {message}") total_tokens = get_token_count([message], model) - verbose_logger.debug( - f"total_tokens: {total_tokens}, tokens_needed: {tokens_needed}" - ) + verbose_logger.debug(f"total_tokens: {total_tokens}, tokens_needed: {tokens_needed}") if total_tokens <= tokens_needed: break @@ -7621,9 +6727,7 @@ def trim_messages( messages = [message for message in messages if message["role"] != "system"] verbose_logger.debug(f"Processed system message: {system_message_event}") - final_messages = process_messages( - messages=messages, max_tokens=max_tokens, model=model - ) + final_messages = process_messages(messages=messages, max_tokens=max_tokens, model=model) verbose_logger.debug(f"Processed messages: {final_messages}") # Add system message to the beginning of the final messages @@ -7633,19 +6737,13 @@ def trim_messages( if len(tool_messages) > 0: final_messages.extend(tool_messages) - verbose_logger.debug( - f"Final messages: {final_messages}, return_response_tokens: {return_response_tokens}" - ) - if ( - return_response_tokens - ): # if user wants token count with new trimmed messages + verbose_logger.debug(f"Final messages: {final_messages}, return_response_tokens: {return_response_tokens}") + if return_response_tokens: # if user wants token count with new trimmed messages response_tokens = max_tokens - get_token_count(final_messages, model) return final_messages, response_tokens return final_messages except Exception as e: # [NON-Blocking, if error occurs just return final_messages - verbose_logger.exception( - "Got exception while token trimming - {}".format(str(e)) - ) + verbose_logger.exception("Got exception while token trimming - {}".format(str(e))) return original_messages @@ -7659,11 +6757,7 @@ class AvailableModelsCache(InMemoryCache): def _get_env_hash(self) -> str: """Create a hash of relevant environment variables""" - env_vars = { - k: v - for k, v in os.environ.items() - if k.startswith(("OPENAI", "ANTHROPIC", "AZURE", "AWS")) - } + env_vars = {k: v for k, v in os.environ.items() if k.startswith(("OPENAI", "ANTHROPIC", "AZURE", "AWS"))} return str(hash(frozenset(env_vars.items()))) def _check_env_changed(self) -> bool: @@ -7738,10 +6832,7 @@ def _infer_valid_provider_from_env_vars( # PROVIDER_API_KEY. Example: OPENAI_API_KEY, COHERE_API_KEY expected_provider_key_1 = f"{env_provider_1.upper()}_API_KEY" expected_provider_key_2 = f"{env_provider_2.upper()}_API_KEY" - if ( - expected_provider_key_1 in environ_keys - or expected_provider_key_2 in environ_keys - ): + if expected_provider_key_1 in environ_keys or expected_provider_key_2 in environ_keys: # key is set valid_providers.append(provider) @@ -7754,9 +6845,7 @@ def _get_valid_models_from_provider_api( litellm_params: Optional[LiteLLM_Params] = None, ) -> List[str]: try: - cached_result = _model_cache.get_cached_model_info( - custom_llm_provider, litellm_params - ) + cached_result = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params) if cached_result is not None: return cached_result @@ -7805,9 +6894,7 @@ def get_valid_models( litellm_params.api_base = api_base ################################# - check_provider_endpoint = ( - check_provider_endpoint or litellm.check_provider_endpoint - ) + check_provider_endpoint = check_provider_endpoint or litellm.check_provider_endpoint # get keys set in .env valid_providers: List[str] = [] @@ -7830,11 +6917,7 @@ def get_valid_models( if provider == "azure": valid_models.append("Azure-LLM") - elif ( - provider_config is not None - and check_provider_endpoint - and provider is not None - ): + elif provider_config is not None and check_provider_endpoint and provider is not None: valid_models.extend( _get_valid_models_from_provider_api( provider_config, @@ -7843,9 +6926,7 @@ def get_valid_models( ) ) else: - models_for_provider = copy.deepcopy( - litellm.models_by_provider.get(provider, []) - ) + models_for_provider = copy.deepcopy(litellm.models_by_provider.get(provider, [])) valid_models.extend(models_for_provider) return valid_models @@ -7859,17 +6940,9 @@ def print_args_passed_to_litellm(original_function, args, kwargs): return try: # we've already printed this for acompletion, don't print for completion - if ( - "acompletion" in kwargs - and kwargs["acompletion"] is True - and original_function.__name__ == "completion" - ): + if "acompletion" in kwargs and kwargs["acompletion"] is True and original_function.__name__ == "completion": return - elif ( - "aembedding" in kwargs - and kwargs["aembedding"] is True - and original_function.__name__ == "embedding" - ): + elif "aembedding" in kwargs and kwargs["aembedding"] is True and original_function.__name__ == "embedding": return elif ( "aimg_generation" in kwargs @@ -7887,17 +6960,11 @@ def print_args_passed_to_litellm(original_function, args, kwargs): "\033[92mRequest to litellm:\033[0m", ) if args and kwargs: - print_verbose( - f"\033[92mlitellm.{original_function.__name__}({args_str}, {kwargs_str})\033[0m" - ) + print_verbose(f"\033[92mlitellm.{original_function.__name__}({args_str}, {kwargs_str})\033[0m") elif args: - print_verbose( - f"\033[92mlitellm.{original_function.__name__}({args_str})\033[0m" - ) + print_verbose(f"\033[92mlitellm.{original_function.__name__}({args_str})\033[0m") elif kwargs: - print_verbose( - f"\033[92mlitellm.{original_function.__name__}({kwargs_str})\033[0m" - ) + print_verbose(f"\033[92mlitellm.{original_function.__name__}({kwargs_str})\033[0m") else: print_verbose(f"\033[92mlitellm.{original_function.__name__}()\033[0m") print_verbose("\n") # new line after @@ -7908,9 +6975,7 @@ def print_args_passed_to_litellm(original_function, args, kwargs): def get_logging_id(start_time, response_obj): try: - response_id = ( - "time-" + start_time.strftime("%H-%M-%S-%f") + "_" + response_obj.get("id") - ) + response_id = "time-" + start_time.strftime("%H-%M-%S-%f") + "_" + response_obj.get("id") return response_id except Exception: return None @@ -7929,9 +6994,7 @@ def _get_base_model_from_metadata(model_call_details=None): _get_base_model_from_litellm_call_metadata = getattr( sys.modules[__name__], "_get_base_model_from_litellm_call_metadata" ) - base_model_from_metadata = _get_base_model_from_litellm_call_metadata( - metadata=metadata - ) + base_model_from_metadata = _get_base_model_from_litellm_call_metadata(metadata=metadata) if base_model_from_metadata is not None: return base_model_from_metadata @@ -7948,12 +7011,8 @@ class ModelResponseIterator: def __init__(self, model_response: ModelResponse, convert_to_delta: bool = False): if convert_to_delta is True: _stream_response = ModelResponseStream() - _stream_response.choices[0].delta.content = model_response.choices[ - 0 - ].message.content # type: ignore - self.model_response: Union[ModelResponse, ModelResponseStream] = ( - _stream_response - ) + _stream_response.choices[0].delta.content = model_response.choices[0].message.content # type: ignore + self.model_response: Union[ModelResponse, ModelResponseStream] = _stream_response else: self.model_response = model_response self.is_done = False @@ -8122,9 +7181,7 @@ def any_assistant_message_has_thinking_blocks( for message in messages: if message.get("role") == "assistant": thinking_blocks = message.get("thinking_blocks") - if thinking_blocks is not None and ( - not hasattr(thinking_blocks, "__len__") or len(thinking_blocks) > 0 - ): + if thinking_blocks is not None and (not hasattr(thinking_blocks, "__len__") or len(thinking_blocks) > 0): return True return False @@ -8158,9 +7215,7 @@ def last_assistant_with_tool_calls_has_no_thinking_blocks( # Check if it has thinking_blocks thinking_blocks = last_assistant_with_tools.get("thinking_blocks") - return thinking_blocks is None or ( - hasattr(thinking_blocks, "__len__") and len(thinking_blocks) == 0 - ) + return thinking_blocks is None or (hasattr(thinking_blocks, "__len__") and len(thinking_blocks) == 0) def add_dummy_tool(custom_llm_provider: str) -> List[ChatCompletionToolParam]: @@ -8209,9 +7264,7 @@ def convert_to_dict(message: Union[BaseModel, dict]) -> dict: elif isinstance(message, dict): return message else: - raise TypeError( - f"Invalid message type: {type(message)}. Expected dict or Pydantic model." - ) + raise TypeError(f"Invalid message type: {type(message)}. Expected dict or Pydantic model.") def convert_list_message_to_dict(messages: List): @@ -8309,9 +7362,7 @@ def validate_chat_completion_user_messages(messages: List[AllMessageValues]): for item in user_content: if isinstance(item, dict): if item.get("type") not in ValidUserMessageContentTypes: - raise Exception( - f"invalid content type={item.get('type')}" - ) + raise Exception(f"invalid content type={item.get('type')}") except Exception as e: if isinstance(e, KeyError): raise Exception( @@ -8346,10 +7397,7 @@ def validate_chat_completion_tool_choice( return tool_choice elif isinstance(tool_choice, dict): # Handle Cursor IDE format: {"type": "auto"} -> return as-is - if ( - tool_choice.get("type") in ["auto", "none", "required"] - and "function" not in tool_choice - ): + if tool_choice.get("type") in ["auto", "none", "required"] and "function" not in tool_choice: return tool_choice # Standard OpenAI format: {"type": "function", "function": {...}} @@ -8376,11 +7424,7 @@ def validate_openai_optional_params( Returns: Validated stop parameter (truncated to 4 elements if needed) """ - if ( - stop is not None - and isinstance(stop, list) - and not litellm.disable_stop_sequence_limit - ): + if stop is not None and isinstance(stop, list) and not litellm.disable_stop_sequence_limit: # Truncate to 4 elements if more are provided as openai only supports up to 4 stop sequences if len(stop) > 4: stop = stop[:4] @@ -8391,9 +7435,7 @@ def validate_openai_optional_params( @lru_cache(maxsize=1) def _get_bundled_model_cost_map() -> Dict[str, Any]: try: - model_cost_path = resources.files("litellm").joinpath( - "model_prices_and_context_window_backup.json" - ) + model_cost_path = resources.files("litellm").joinpath("model_prices_and_context_window_backup.json") return json.loads(model_cost_path.read_text()) except Exception: return {} @@ -8685,15 +7727,11 @@ class ProviderConfigManager: # Handle Azure before the generic map so base_model can be threaded through if provider == LlmProviders.AZURE: - return ProviderConfigManager._get_azure_config( - model=model, base_model=base_model - ) + return ProviderConfigManager._get_azure_config(model=model, base_model=base_model) # Initialize provider config map lazily (avoids circular imports) if ProviderConfigManager._PROVIDER_CONFIG_MAP is None: - ProviderConfigManager._PROVIDER_CONFIG_MAP = ( - ProviderConfigManager._build_provider_config_map() - ) + ProviderConfigManager._PROVIDER_CONFIG_MAP = ProviderConfigManager._build_provider_config_map() # O(1) dictionary lookup — Python classes first (custom overrides take priority) config_entry = ProviderConfigManager._PROVIDER_CONFIG_MAP.get(provider) @@ -8723,9 +7761,7 @@ class ProviderConfigManager: ) -> Optional[BaseEmbeddingConfig]: if ( litellm.LlmProviders.VOYAGE == provider - and litellm.VoyageContextualEmbeddingConfig.is_contextualized_embeddings( - model - ) + and litellm.VoyageContextualEmbeddingConfig.is_contextualized_embeddings(model) ): return litellm.VoyageContextualEmbeddingConfig() elif ( @@ -8749,10 +7785,7 @@ class ProviderConfigManager: from litellm.llms.oci.embed.transformation import OCIEmbedConfig return OCIEmbedConfig() - elif ( - litellm.LlmProviders.COHERE == provider - or litellm.LlmProviders.COHERE_CHAT == provider - ): + elif litellm.LlmProviders.COHERE == provider or litellm.LlmProviders.COHERE_CHAT == provider: from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig return CohereEmbeddingConfig() @@ -8815,10 +7848,7 @@ class ProviderConfigManager: api_base: Optional[str], present_version_params: List[str], ) -> BaseRerankConfig: - if ( - litellm.LlmProviders.COHERE == provider - or litellm.LlmProviders.COHERE_CHAT == provider - ): + if litellm.LlmProviders.COHERE == provider or litellm.LlmProviders.COHERE_CHAT == provider: if should_use_cohere_v1_client(api_base, present_version_params): return litellm.CohereRerankConfig() else: @@ -8862,9 +7892,7 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> Optional[BaseAnthropicMessagesConfig]: - return ProviderConfigManager._get_provider_anthropic_messages_config_cached( - model=model, provider=provider - ) + return ProviderConfigManager._get_provider_anthropic_messages_config_cached(model=model, provider=provider) @staticmethod @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) @@ -8995,9 +8023,7 @@ class ProviderConfigManager: from litellm.llms.openai_like.json_loader import JSONProviderRegistry # Resolve provider string for JSON lookup - provider_str = ( - provider.value if isinstance(provider, LlmProviders) else str(provider) - ) + provider_str = provider.value if isinstance(provider, LlmProviders) else str(provider) # Try to convert to enum for Python class lookup first. # Python classes take priority over JSON (they have custom overrides). @@ -9011,16 +8037,12 @@ class ProviderConfigManager: pass # Check Python classes first (custom overrides take priority) - result = ProviderConfigManager._get_python_responses_api_config( - provider_enum, model - ) + result = ProviderConfigManager._get_python_responses_api_config(provider_enum, model) if result is not None: return result # Fall back to JSON providers (generic OpenAI-compatible) - if JSONProviderRegistry.exists( - provider_str - ) and JSONProviderRegistry.supports_responses_api(provider_str): + if JSONProviderRegistry.exists(provider_str) and JSONProviderRegistry.supports_responses_api(provider_str): provider_config = JSONProviderRegistry.get(provider_str) if provider_config is not None: return create_responses_config_class(provider_config)() @@ -9043,10 +8065,7 @@ class ProviderConfigManager: # Note: GPT models (gpt-3.5, gpt-4, gpt-5, etc.) support temperature parameter # O-series models (o1, o3) do not contain "gpt" and have different parameter restrictions is_gpt_model = model and "gpt" in model.lower() - is_o_series = model and ( - "o_series" in model.lower() - or (supports_reasoning(model) and not is_gpt_model) - ) + is_o_series = model and ("o_series" in model.lower() or (supports_reasoning(model) and not is_gpt_model)) if is_o_series: return litellm.AzureOpenAIOSeriesResponsesAPIConfig() @@ -9099,8 +8118,7 @@ class ProviderConfigManager: if not model or not mantle_supports_responses(model, litellm.model_cost): return None return litellm.BedrockMantleResponsesAPIConfig( - use_openai_path=mantle_base_segment(model, litellm.model_cost) - == "openai/v1" + use_openai_path=mantle_base_segment(model, litellm.model_cost) == "openai/v1" ) return None @@ -9850,17 +8868,12 @@ def get_end_user_id_for_cost_tracking( service_type: "litellm_logging" or "prometheus" - used to allow prometheus only disable cost tracking. """ - get_litellm_metadata_from_kwargs = getattr( - sys.modules[__name__], "get_litellm_metadata_from_kwargs" - ) - _metadata = cast( - dict, get_litellm_metadata_from_kwargs(dict(litellm_params=litellm_params)) - ) + get_litellm_metadata_from_kwargs = getattr(sys.modules[__name__], "get_litellm_metadata_from_kwargs") + _metadata = cast(dict, get_litellm_metadata_from_kwargs(dict(litellm_params=litellm_params))) end_user_id = cast( Optional[str], - litellm_params.get("user_api_key_end_user_id") - or _metadata.get("user_api_key_end_user_id"), + litellm_params.get("user_api_key_end_user_id") or _metadata.get("user_api_key_end_user_id"), ) if litellm.disable_end_user_cost_tracking: return None @@ -9875,17 +8888,13 @@ def get_end_user_id_for_cost_tracking( return end_user_id -def should_use_cohere_v1_client( - api_base: Optional[str], present_version_params: List[str] -): +def should_use_cohere_v1_client(api_base: Optional[str], present_version_params: List[str]): if not api_base: return False uses_v1_params = ("max_chunks_per_doc" in present_version_params) and ( "max_tokens_per_doc" not in present_version_params ) - return api_base.endswith("/v1/rerank") or ( - uses_v1_params and not api_base.endswith("/v2/rerank") - ) + return api_base.endswith("/v1/rerank") or (uses_v1_params and not api_base.endswith("/v2/rerank")) def is_prompt_caching_valid_prompt( @@ -9902,9 +8911,7 @@ def is_prompt_caching_valid_prompt( try: if messages is None and tools is None: return False - if custom_llm_provider is not None and not model.startswith( - custom_llm_provider - ): + if custom_llm_provider is not None and not model.startswith(custom_llm_provider): model = custom_llm_provider + "/" + model token_count = token_counter( messages=messages, @@ -9985,11 +8992,7 @@ def _add_path_to_api_base(api_base: str, ending_path: str) -> str: def get_standard_openai_params(params: dict) -> dict: - return { - k: v - for k, v in params.items() - if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS and v is not None - } + return {k: v for k, v in params.items() if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS and v is not None} def get_non_default_completion_params(kwargs: dict) -> dict: @@ -10072,9 +9075,7 @@ def add_openai_metadata( return None # Only include non-hidden parameters visible_metadata: Dict[str, str] = { - str(k): v - for k, v in metadata.items() - if k != "hidden_params" and isinstance(v, str) + str(k): v for k, v in metadata.items() if k != "hidden_params" and isinstance(v, str) } # max 16 keys allowed by openai - trim down to 16 @@ -10141,9 +9142,7 @@ def return_raw_request(endpoint: CallTypes, kwargs: dict) -> RawRequestTypedDict except Exception as e: received_exception = str(e) - raw_request_typed_dict = litellm_logging_obj.model_call_details.get( - "raw_request_typed_dict" - ) + raw_request_typed_dict = litellm_logging_obj.model_call_details.get("raw_request_typed_dict") if raw_request_typed_dict: return cast(RawRequestTypedDict, raw_request_typed_dict) else: diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 807f1fe95f5..4c4ca1b8a12 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -12,9 +12,7 @@ from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../../..") -) # Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../../../../../../..")) # Adds the parent directory to the system path from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms.anthropic.chat.guardrail_translation.handler import ( @@ -51,9 +49,7 @@ class MockDynamicGuardrail(CustomGuardrail): input_type: Literal["request", "response"], logging_obj: Optional[Any] = None, ) -> GenericGuardrailAPIInputs: - self.dynamic_params = self.get_guardrail_dynamic_request_body_params( - request_data - ) + self.dynamic_params = self.get_guardrail_dynamic_request_body_params(request_data) return inputs @@ -103,17 +99,11 @@ class TestAnthropicMessagesHandlerInputProcessing: data = { "model": "claude-3-5-sonnet-20241022", "messages": [{"role": "user", "content": "hello"}], - "litellm_metadata": { - "guardrails": [ - {"cygnal-monitor": {"extra_body": {"policy_id": "policy-123"}}} - ] - }, + "litellm_metadata": {"guardrails": [{"cygnal-monitor": {"extra_body": {"policy_id": "policy-123"}}}]}, } with patch("litellm.proxy.proxy_server.premium_user", True): - await handler.process_input_messages( - data=data, guardrail_to_apply=guardrail - ) + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) assert data.get("litellm_metadata", {}).get("guardrails") assert guardrail.dynamic_params == {"policy_id": "policy-123"} @@ -216,9 +206,7 @@ class TestAnthropicMessagesHandlerInputProcessing: # Mock _check_streaming_has_ended to return False (stream not ended) with ( patch.object(handler, "_check_streaming_has_ended", return_value=False), - patch.object( - handler, "get_streaming_string_so_far", return_value="partial text" - ), + patch.object(handler, "get_streaming_string_so_far", return_value="partial text"), ): responses_so_far = [b"data: some chunk"] @@ -249,9 +237,7 @@ class TestAnthropicMessagesHandlerInputProcessing: data = { "model": "claude-opus-4-6", - "messages": [ - {"role": "user", "content": "What is the weather in San Francisco?"} - ], + "messages": [{"role": "user", "content": "What is the weather in San Francisco?"}], "tools": [ { "type": "tool_search_tool_regex_20251119", @@ -295,6 +281,122 @@ class TestAnthropicMessagesHandlerInputProcessing: assert "input_schema" in tools[1] +class MockStructuredMessagesGuardrail(CustomGuardrail): + """Mock guardrail that returns compressed structured_messages.""" + + def __init__(self, guardrail_name: str, compressed_messages: List[Any]): + super().__init__(guardrail_name=guardrail_name) + self.compressed_messages = compressed_messages + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + result = dict(inputs) + result["structured_messages"] = self.compressed_messages + return GenericGuardrailAPIInputs(**result) + + +class TestAnthropicStructuredMessagesApplied: + """Test that structured_messages from guardrail response are applied back in Anthropic format.""" + + @pytest.mark.asyncio + async def test_structured_messages_replaced_in_anthropic_format(self): + """When guardrail returns structured_messages (OpenAI format), they must be + converted back to Anthropic format and written to data['messages'].""" + handler = AnthropicMessagesHandler() + + compressed_openai_messages = [ + {"role": "user", "content": "compressed content"}, + ] + guardrail = MockStructuredMessagesGuardrail( + guardrail_name="test-compressor", + compressed_messages=compressed_openai_messages, + ) + + data: dict = { + "model": "claude-3-5-sonnet-20241022", + "messages": [ + {"role": "user", "content": "original long content that gets compressed"}, + ], + } + + result = await handler.process_input_messages( + data=data, guardrail_to_apply=guardrail, litellm_logging_obj=MagicMock() + ) + + messages = result["messages"] + assert len(messages) == 1 + assert messages[0]["role"] == "user" + content = messages[0]["content"] + if isinstance(content, str): + assert content == "compressed content" + elif isinstance(content, list): + assert any(block.get("text") == "compressed content" for block in content if isinstance(block, dict)) + + @pytest.mark.asyncio + async def test_no_structured_messages_falls_back_to_text_patching(self): + """When guardrail returns no structured_messages, the original text-patch path runs.""" + handler = AnthropicMessagesHandler() + guardrail = MockPassThroughGuardrail(guardrail_name="test") + + original_content = "original content" + data: dict = { + "model": "claude-3-5-sonnet-20241022", + "messages": [{"role": "user", "content": original_content}], + } + + result = await handler.process_input_messages( + data=data, guardrail_to_apply=guardrail, litellm_logging_obj=MagicMock() + ) + + messages = result["messages"] + assert len(messages) == 1 + content = messages[0]["content"] + if isinstance(content, str): + assert content == original_content + + +class TestFunctionSetupLitellmMetadataReference: + """Regression test: guardrail info written to litellm_metadata must appear in + standard logging payload for endpoints that use litellm_metadata (e.g. /v1/messages).""" + + def test_litellm_metadata_reference_not_copy(self): + """litellm_params['metadata'] must be the same object as kwargs['litellm_metadata'] + so mutations by guardrails after function_setup are visible at logging time.""" + import litellm + from datetime import datetime + + litellm_metadata_dict: dict = {"user_api_key": "test-key"} + kwargs = { + "model": "claude-3-5-sonnet-20241022", + "messages": [{"role": "user", "content": "hello"}], + "litellm_call_id": "test-call-id", + "litellm_metadata": litellm_metadata_dict, + } + + logging_obj, _ = litellm.utils.function_setup( + original_function="anthropic_messages", + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), + **kwargs, + ) + + litellm_metadata_dict["standard_logging_guardrail_information"] = [ + {"guardrail_name": "test-guardrail", "guardrail_status": "success"} + ] + + metadata_in_logging = logging_obj.litellm_params.get("metadata", {}) + assert "standard_logging_guardrail_information" in metadata_in_logging, ( + "guardrail info written to litellm_metadata after function_setup must be " + "visible in litellm_params['metadata'] — check function_setup uses a " + "reference not a copy for litellm_metadata endpoints" + ) + + if __name__ == "__main__": # Run the tests pytest.main([__file__, "-v"])