diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ab034d9f51b..6d87e0b5997 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16094,6 +16094,181 @@ "output_cost_per_token": 0.0, "output_vector_size": 2560 }, + "gmi/anthropic/claude-opus-4.5": { + "input_cost_per_token": 5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "supports_function_calling": true, + "supports_vision": true + }, + "gmi/anthropic/claude-sonnet-4.5": { + "input_cost_per_token": 3e-06, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_function_calling": true, + "supports_vision": true + }, + "gmi/anthropic/claude-sonnet-4": { + "input_cost_per_token": 3e-06, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_function_calling": true, + "supports_vision": true + }, + "gmi/anthropic/claude-opus-4": { + "input_cost_per_token": 1.5e-05, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "supports_function_calling": true, + "supports_vision": true + }, + "gmi/openai/gpt-5.2": { + "input_cost_per_token": 1.75e-06, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "supports_function_calling": true + }, + "gmi/openai/gpt-5.1": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true + }, + "gmi/openai/gpt-5": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true + }, + "gmi/openai/gpt-4o": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_vision": true + }, + "gmi/openai/gpt-4o-mini": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_function_calling": true, + "supports_vision": true + }, + "gmi/deepseek-ai/DeepSeek-V3.2": { + "input_cost_per_token": 2.8e-07, + "litellm_provider": "gmi", + "max_input_tokens": 163840, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 4e-07, + "supports_function_calling": true + }, + "gmi/deepseek-ai/DeepSeek-V3-0324": { + "input_cost_per_token": 2.8e-07, + "litellm_provider": "gmi", + "max_input_tokens": 163840, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 8.8e-07, + "supports_function_calling": true + }, + "gmi/google/gemini-3-pro-preview": { + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "supports_function_calling": true, + "supports_vision": true + }, + "gmi/google/gemini-3-flash-preview": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_vision": true + }, + "gmi/moonshotai/Kimi-K2-Thinking": { + "input_cost_per_token": 8e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.2e-06 + }, + "gmi/MiniMaxAI/MiniMax-M2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 196608, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.2e-06 + }, + "gmi/Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": { + "input_cost_per_token": 3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.4e-06, + "supports_vision": true + }, + "gmi/zai-org/GLM-4.7-FP8": { + "input_cost_per_token": 4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 202752, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 2e-06 + }, "google.gemma-3-12b-it": { "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", diff --git a/litellm/utils.py b/litellm/utils.py index ad9e36795af..2a973dddff6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -14,6 +14,7 @@ import io import itertools import json import logging +import mimetypes import os import random # type: ignore import re @@ -771,13 +772,15 @@ def function_setup( # noqa: PLR0915 function_id: Optional[str] = kwargs["id"] if "id" in kwargs else None ## LAZY LOAD COROUTINE CHECKER ## - get_coroutine_checker_fn = getattr(sys.modules[__name__], "get_coroutine_checker") + get_coroutine_checker_fn = getattr( + sys.modules[__name__], "get_coroutine_checker" + ) coroutine_checker = get_coroutine_checker_fn() ## DYNAMIC CALLBACKS ## - dynamic_callbacks: Optional[ - List[Union[str, Callable, "CustomLogger"]] - ] = kwargs.pop("callbacks", None) + dynamic_callbacks: Optional[List[Union[str, Callable, "CustomLogger"]]] = ( + kwargs.pop("callbacks", None) + ) all_callbacks = get_dynamic_callbacks(dynamic_callbacks=dynamic_callbacks) if len(all_callbacks) > 0: @@ -1449,6 +1452,7 @@ def client(original_function): # noqa: PLR0915 logging_obj, kwargs = function_setup( original_function.__name__, rules_obj, start_time, *args, **kwargs ) + logging_obj = cast(LiteLLMLoggingObject, logging_obj) ## LOAD CREDENTIALS load_credentials_from_list(kwargs) kwargs["litellm_logging_obj"] = logging_obj @@ -1660,9 +1664,9 @@ def client(original_function): # noqa: PLR0915 exception=e, retry_policy=kwargs.get("retry_policy"), ) - kwargs[ - "retry_policy" - ] = reset_retry_policy() # prevent infinite loops + kwargs["retry_policy"] = ( + reset_retry_policy() + ) # prevent infinite loops litellm.num_retries = ( None # set retries to None to prevent infinite loops ) @@ -1709,9 +1713,9 @@ def client(original_function): # noqa: PLR0915 exception=e, retry_policy=kwargs.get("retry_policy"), ) - kwargs[ - "retry_policy" - ] = reset_retry_policy() # prevent infinite loops + kwargs["retry_policy"] = ( + reset_retry_policy() + ) # prevent infinite loops litellm.num_retries = ( None # set retries to None to prevent infinite loops ) @@ -1767,6 +1771,7 @@ def client(original_function): # noqa: PLR0915 original_function.__name__, rules_obj, start_time, *args, **kwargs ) + logging_obj = cast(LiteLLMLoggingObject, logging_obj) modified_kwargs = await async_pre_call_deployment_hook(kwargs, call_type) if modified_kwargs is not None: kwargs = modified_kwargs @@ -3640,10 +3645,10 @@ def pre_process_non_default_params( if "response_format" in non_default_params: if provider_config is not None: - non_default_params[ - "response_format" - ] = provider_config.get_json_schema_from_pydantic_object( - response_format=non_default_params["response_format"] + non_default_params["response_format"] = ( + provider_config.get_json_schema_from_pydantic_object( + response_format=non_default_params["response_format"] + ) ) else: non_default_params["response_format"] = type_to_response_format_param( @@ -3772,16 +3777,16 @@ def pre_process_optional_params( True # so that main.py adds the function call to the prompt ) if "tools" in non_default_params: - optional_params[ - "functions_unsupported_model" - ] = non_default_params.pop("tools") + optional_params["functions_unsupported_model"] = ( + non_default_params.pop("tools") + ) non_default_params.pop( "tool_choice", None ) # causes ollama requests to hang elif "functions" in non_default_params: - optional_params[ - "functions_unsupported_model" - ] = non_default_params.pop("functions") + optional_params["functions_unsupported_model"] = ( + non_default_params.pop("functions") + ) elif ( litellm.add_function_to_prompt ): # if user opts to add it to prompt instead @@ -4937,9 +4942,9 @@ def get_response_string(response_obj: Union[ModelResponse, ModelResponseStream]) return delta if isinstance(delta, str) else "" # Handle standard ModelResponse and ModelResponseStream - _choices: Union[ - List[Union[Choices, StreamingChoices]], List[StreamingChoices] - ] = response_obj.choices + _choices: Union[List[Union[Choices, StreamingChoices]], List[StreamingChoices]] = ( + response_obj.choices + ) # Use list accumulation to avoid O(n^2) string concatenation across choices response_parts: List[str] = [] @@ -7585,11 +7590,82 @@ def convert_list_message_to_dict(messages: List): return new_messages +def format_base64_as_data_uri( + base64_data: str, filename: Optional[str] = None, format: Optional[str] = None +) -> str: + """ + Convert raw base64 data to a properly formatted data URI for OpenAI. + + Args: + base64_data: Raw base64 encoded string (without data URI prefix) + filename: Optional filename to infer MIME type from extension + format: Optional explicit format/MIME type (e.g., "image/jpeg", "application/pdf") + + Returns: + Formatted data URI string like "data:image/jpeg;base64,{base64_data}" + + Examples: + >>> format_base64_as_data_uri("iVBORw0KG...", filename="image.png") + "data:image/png;base64,iVBORw0KG..." + + >>> format_base64_as_data_uri("JVBERi0xL...", format="application/pdf") + "data:application/pdf;base64,JVBERi0xL..." + """ + # Strip any existing data URI prefix if present + if base64_data.startswith("data:"): + # Already formatted, return as-is + return base64_data + + # Remove any whitespace from base64 data + base64_data = base64_data.strip() + + # Determine MIME type + mime_type = None + + # Priority 1: Use explicit format if provided + if format: + # If format is already a MIME type (contains /), use it directly + if "/" in format: + mime_type = format + else: + # Otherwise treat it as a file extension + mime_type = mimetypes.guess_type(f"file.{format}")[0] + + # Priority 2: Infer from filename extension + if not mime_type and filename: + guessed_type = mimetypes.guess_type(filename)[0] + if guessed_type: + mime_type = guessed_type + + # Priority 3: Try to infer from base64 header (magic bytes) + if not mime_type: + # Check common file signatures in base64 + if base64_data.startswith("iVBORw0KG"): + mime_type = "image/png" + elif base64_data.startswith("/9j/"): + mime_type = "image/jpeg" + elif base64_data.startswith("JVBERi0"): + mime_type = "application/pdf" + elif base64_data.startswith("R0lGOD"): + mime_type = "image/gif" + elif base64_data.startswith("UklGR"): + mime_type = "image/webp" + + # Default fallback + if not mime_type: + # Default to application/octet-stream for unknown types + mime_type = "application/octet-stream" + + # Return formatted data URI + return f"data:{mime_type};base64,{base64_data}" + + def validate_and_fix_openai_messages(messages: List): """ Ensures all messages are valid OpenAI chat completion messages. Handles missing role for assistant messages. + Converts raw base64 file data to proper data URI format. """ new_messages = [] for message in messages: @@ -7598,6 +7674,24 @@ def validate_and_fix_openai_messages(messages: List): if message.get("tool_calls"): message["tool_calls"] = jsonify_tools(tools=message["tool_calls"]) + # Handle file content with raw base64 data + content = message.get("content") + if isinstance(content, list): + for element in content: + if isinstance(element, dict) and element.get("type") == "file": + file_obj = element.get("file", {}) + file_data = file_obj.get("file_data") + + # Check if file_data exists and doesn't already have data URI prefix + if file_data and not file_data.startswith("data:"): + filename = file_obj.get("filename") + file_format = file_obj.get("format") + + # Convert raw base64 to properly formatted data URI + file_obj["file_data"] = format_base64_as_data_uri( + base64_data=file_data, filename=filename, format=file_format + ) + convert_msg_to_dict = cast(AllMessageValues, convert_to_dict(message)) cleaned_message = cleanup_none_field_in_message(message=convert_msg_to_dict) new_messages.append(cleaned_message) @@ -7707,25 +7801,29 @@ def validate_chat_completion_tool_choice( f"Invalid tool choice, tool_choice={tool_choice}. Got={type(tool_choice)}. Expecting str, or dict. Please ensure tool_choice follows the OpenAI tool_choice spec" ) -def validate_openai_optional_params( - stop: Optional[Union[str, List[str]]] = None, - **kwargs -) -> Optional[Union[str, List[str]]]: - """ - Validates and fixes OpenAI optional parameters. - - Args: - stop: Stop sequences (string or list of strings) - **kwargs: Additional optional parameters - - Returns: - Validated stop parameter (truncated to 4 elements if needed) - """ - if stop is not None and isinstance(stop, list) and not litellm.disable_stop_sequence_limit: + +def validate_openai_optional_params( + stop: Optional[Union[str, List[str]]] = None, **kwargs +) -> Optional[Union[str, List[str]]]: + """ + Validates and fixes OpenAI optional parameters. + + Args: + stop: Stop sequences (string or list of strings) + **kwargs: Additional optional parameters + + Returns: + Validated stop parameter (truncated to 4 elements if needed) + """ + if ( + stop is not None + and isinstance(stop, list) + and not litellm.disable_stop_sequence_limit + ): # Truncate to 4 elements if more are provided as openai only supports up to 4 stop sequences - if len(stop) > 4: - stop = stop[:4] - + if len(stop) > 4: + stop = stop[:4] + return stop @@ -8686,9 +8784,9 @@ class ProviderConfigManager: """ Get Search configuration for a given provider. """ + from litellm.llms.brave.search.transformation import BraveSearchConfig from litellm.llms.dataforseo.search.transformation import DataForSEOSearchConfig from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig - from litellm.llms.brave.search.transformation import BraveSearchConfig from litellm.llms.firecrawl.search.transformation import FirecrawlSearchConfig from litellm.llms.google_pse.search.transformation import GooglePSESearchConfig from litellm.llms.linkup.search.transformation import LinkupSearchConfig diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 7c0db41d13a..b8d7d6f3d14 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -3954,3 +3954,32 @@ def test_bedrock_openai_error_handling(): assert exc_info.value.status_code == 422 print("✓ Error handling works correctly") + + +def test_aaabedrock_completion_pdf(): + import litellm + + import base64 + + # Read a PDF file + with open("./fixtures/dummy.pdf", "rb") as f: + pdf_base64 = base64.b64encode(f.read()).decode("utf-8") + + response = litellm.completion( + model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this document?"}, + { + "type": "file", + "file": { + "file_data": pdf_base64, + "filename": "document.pdf", + }, + }, + ], + } + ], + )