diff --git a/docs/my-website/docs/completion/drop_params.md b/docs/my-website/docs/completion/drop_params.md index e79a88e14bb..590d9a45955 100644 --- a/docs/my-website/docs/completion/drop_params.md +++ b/docs/my-website/docs/completion/drop_params.md @@ -107,4 +107,76 @@ response = litellm.completion( -**additional_drop_params**: List or null - Is a list of openai params you want to drop when making a call to the model. \ No newline at end of file +**additional_drop_params**: List or null - Is a list of openai params you want to drop when making a call to the model. + +## Specify allowed openai params in a request + +Tell litellm to allow specific openai params in a request. Use this if you get a `litellm.UnsupportedParamsError` and want to allow a param. LiteLLM will pass the param as is to the model. + + + + + + +In this example we pass `allowed_openai_params=["tools"]` to allow the `tools` param. + +```python showLineNumbers title="Pass allowed_openai_params to LiteLLM Python SDK" +await litellm.acompletion( + model="azure/o_series/", + api_key="xxxxx", + api_base=api_base, + messages=[{"role": "user", "content": "Hello! return a json object"}], + tools=[{"type": "function", "function": {"name": "get_current_time", "description": "Get the current time in a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name, e.g. San Francisco"}}, "required": ["location"]}}}] + allowed_openai_params=["tools"], +) +``` + + + +When using litellm proxy you can pass `allowed_openai_params` in two ways: + +1. Dynamically pass `allowed_openai_params` in a request +2. Set `allowed_openai_params` on the config.yaml file for a specific model + +#### Dynamically pass allowed_openai_params in a request +In this example we pass `allowed_openai_params=["tools"]` to allow the `tools` param for a request sent to the model set on the proxy. + +```python showLineNumbers title="Dynamically pass allowed_openai_params in a request" +import openai +from openai import AsyncAzureOpenAI + +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "allowed_openai_params": ["tools"] + } +) +``` + +#### Set allowed_openai_params on config.yaml + +You can also set `allowed_openai_params` on the config.yaml file for a specific model. This means that all requests to this deployment are allowed to pass in the `tools` param. + +```yaml showLineNumbers title="Set allowed_openai_params on config.yaml" +model_list: + - model_name: azure-o1-preview + litellm_params: + model: azure/o_series/ + api_key: xxxxx + api_base: https://openai-prod-test.openai.azure.com/openai/deployments/o1/chat/completions?api-version=2025-01-01-preview + allowed_openai_params: ["tools"] +``` + + \ No newline at end of file diff --git a/docs/my-website/docs/proxy/db_deadlocks.md b/docs/my-website/docs/proxy/db_deadlocks.md new file mode 100644 index 00000000000..51052a0bc59 --- /dev/null +++ b/docs/my-website/docs/proxy/db_deadlocks.md @@ -0,0 +1,65 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# High Availability Setup (Resolve DB Deadlocks) + +Resolve any Database Deadlocks you see in high traffic by using this setup + +## What causes the problem? + +LiteLLM writes `UPDATE` and `UPSERT` queries to the DB. When using 10+ instances of LiteLLM, these queries can cause deadlocks since each instance could simultaneously attempt to update the same `user_id`, `team_id`, `key` etc. + +## How the high availability setup fixes the problem +- All instances will write to a Redis queue instead of the DB. +- A single instance will acquire a lock on the DB and flush the redis queue to the DB. + + +## How it works + +### Stage 1. Each instance writes updates to redis + +Each instance will accumlate the spend updates for a key, user, team, etc and write the updates to a redis queue. + + +

+Each instance writes updates to redis +

+ + +### Stage 2. A single instance flushes the redis queue to the DB + +A single instance will acquire a lock on the DB and flush all elements in the redis queue to the DB. + + + +

+A single instance flushes the redis queue to the DB +

+ + +## Usage + +### Required components + +- Redis +- Postgres + +### Setup on LiteLLM config + +You can enable using the redis buffer by setting `use_redis_transaction_buffer: true` in the `general_settings` section of your `proxy_config.yaml` file. + +Note: This setup requires litellm to be connected to a redis instance. + +```yaml showLineNumbers title="litellm proxy_config.yaml" +general_settings: + use_redis_transaction_buffer: true + +litellm_settings: + cache: True + cache_params: + type: redis + supported_call_types: [] # Optional: Set cache for proxy, but not on the actual llm api call +``` + + diff --git a/docs/my-website/img/deadlock_fix_1.png b/docs/my-website/img/deadlock_fix_1.png new file mode 100644 index 00000000000..3dff86a4d20 Binary files /dev/null and b/docs/my-website/img/deadlock_fix_1.png differ diff --git a/docs/my-website/img/deadlock_fix_2.png b/docs/my-website/img/deadlock_fix_2.png new file mode 100644 index 00000000000..345acff586f Binary files /dev/null and b/docs/my-website/img/deadlock_fix_2.png differ diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json index 6c07e67d912..06251b16bbc 100644 --- a/docs/my-website/package-lock.json +++ b/docs/my-website/package-lock.json @@ -12559,9 +12559,10 @@ } }, "node_modules/image-size": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/image-size/-/image-size-1.1.1.tgz", - "integrity": "sha512-541xKlUw6jr/6gGuk92F+mYM5zaFAc5ahphvkqvNe2bQ6gVBkd6bfrmVJ2t4KDAfikAYZyIqTnktX3i6/aQDrQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-1.2.1.tgz", + "integrity": "sha512-rH+46sQJ2dlwfjfhCyNx5thzrv+dtmBIhPHk0zgRUukHzZ/kRueTJXoYYsclBaKcSMBWuGbOFXtioLpzTb5euw==", + "license": "MIT", "dependencies": { "queue": "6.0.2" }, diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 1542edceb5d..1cb3e02707d 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -53,7 +53,7 @@ const sidebars = { { type: "category", label: "Architecture", - items: ["proxy/architecture", "proxy/db_info", "router_architecture", "proxy/user_management_heirarchy", "proxy/jwt_auth_arch", "proxy/image_handling"], + items: ["proxy/architecture", "proxy/db_info", "proxy/db_deadlocks", "router_architecture", "proxy/user_management_heirarchy", "proxy/jwt_auth_arch", "proxy/image_handling"], }, { type: "link", diff --git a/litellm/llms/azure/chat/o_series_transformation.py b/litellm/llms/azure/chat/o_series_transformation.py index 0ca3a28d23e..21aafce7fbf 100644 --- a/litellm/llms/azure/chat/o_series_transformation.py +++ b/litellm/llms/azure/chat/o_series_transformation.py @@ -14,6 +14,7 @@ Translations handled by LiteLLM: from typing import List, Optional +import litellm from litellm import verbose_logger from litellm.types.llms.openai import AllMessageValues from litellm.utils import get_model_info @@ -22,6 +23,27 @@ from ...openai.chat.o_series_transformation import OpenAIOSeriesConfig class AzureOpenAIO1Config(OpenAIOSeriesConfig): + def get_supported_openai_params(self, model: str) -> list: + """ + Get the supported OpenAI params for the Azure O-Series models + """ + all_openai_params = litellm.OpenAIGPTConfig().get_supported_openai_params( + model=model + ) + non_supported_params = [ + "logprobs", + "top_p", + "presence_penalty", + "frequency_penalty", + "top_logprobs", + ] + + o_series_only_param = ["reasoning_effort"] + all_openai_params.extend(o_series_only_param) + return [ + param for param in all_openai_params if param not in non_supported_params + ] + def should_fake_stream( self, model: Optional[str], diff --git a/litellm/main.py b/litellm/main.py index f69454aaad8..56b0aa36716 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1115,6 +1115,7 @@ def completion( # type: ignore # noqa: PLR0915 messages=messages, reasoning_effort=reasoning_effort, thinking=thinking, + allowed_openai_params=kwargs.get("allowed_openai_params"), **non_default_params, ) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 2ee830bca44..52948c927e9 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -13,5 +13,3 @@ litellm_settings: cache_params: type: redis supported_call_types: [] - callbacks: ["prometheus"] - service_callback: ["prometheus_system"] \ No newline at end of file diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8716779d1f8..51a6ed17b16 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1950,6 +1950,7 @@ all_litellm_params = [ "use_in_pass_through", "merge_reasoning_content_in_choices", "litellm_credential_name", + "allowed_openai_params", ] + list(StandardCallbackDynamicParams.__annotations__.keys()) diff --git a/litellm/utils.py b/litellm/utils.py index 1a35e58dc1c..4283cf2df11 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2843,6 +2843,7 @@ def get_optional_params( # noqa: PLR0915 api_version=None, parallel_tool_calls=None, drop_params=None, + allowed_openai_params: Optional[List[str]] = None, reasoning_effort=None, additional_drop_params=None, messages: Optional[List[AllMessageValues]] = None, @@ -2928,6 +2929,7 @@ def get_optional_params( # noqa: PLR0915 "api_version": None, "parallel_tool_calls": None, "drop_params": None, + "allowed_openai_params": None, "additional_drop_params": None, "messages": None, "reasoning_effort": None, @@ -2944,6 +2946,7 @@ def get_optional_params( # noqa: PLR0915 and k != "custom_llm_provider" and k != "api_version" and k != "drop_params" + and k != "allowed_openai_params" and k != "additional_drop_params" and k != "messages" and k in default_params @@ -3053,6 +3056,12 @@ def get_optional_params( # noqa: PLR0915 tool_function["parameters"] = new_parameters def _check_valid_arg(supported_params: List[str]): + """ + Check if the params passed to completion() are supported by the provider + + Args: + supported_params: List[str] - supported params from the litellm config + """ verbose_logger.info( f"\nLiteLLM completion() model= {model}; provider = {custom_llm_provider}" ) @@ -3086,7 +3095,7 @@ def get_optional_params( # noqa: PLR0915 else: raise UnsupportedParamsError( status_code=500, - message=f"{custom_llm_provider} does not support parameters: {unsupported_params}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n", + message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.", ) supported_params = get_supported_openai_params( @@ -3096,7 +3105,14 @@ def get_optional_params( # noqa: PLR0915 supported_params = get_supported_openai_params( model=model, custom_llm_provider="openai" ) - _check_valid_arg(supported_params=supported_params or []) + + supported_params = supported_params or [] + allowed_openai_params = allowed_openai_params or [] + supported_params.extend(allowed_openai_params) + + _check_valid_arg( + supported_params=supported_params or [], + ) ## raise exception if provider doesn't support passed in param if custom_llm_provider == "anthropic": ## check if unsupported param passed in @@ -3735,6 +3751,26 @@ def get_optional_params( # noqa: PLR0915 if k not in default_params.keys(): optional_params[k] = passed_params[k] print_verbose(f"Final returned optional params: {optional_params}") + optional_params = _apply_openai_param_overrides( + optional_params=optional_params, + non_default_params=non_default_params, + allowed_openai_params=allowed_openai_params, + ) + return optional_params + + +def _apply_openai_param_overrides( + optional_params: dict, non_default_params: dict, allowed_openai_params: list +): + """ + If user passes in allowed_openai_params, apply them to optional_params + + These params will get passed as is to the LLM API since the user opted in to passing them in the request + """ + if allowed_openai_params: + for param in allowed_openai_params: + if param not in optional_params: + optional_params[param] = non_default_params.pop(param, None) return optional_params diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 1f2c563f266..bee9d1f7d50 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -167,7 +167,9 @@ class TestAzureOpenAIDalle3(BaseImageGenTest): litellm.set_verbose = True return { "model": "azure/dall-e-3-test", - "api_version": "2023-09-01-preview", + "api_version": "2023-12-01-preview", + "api_base": os.getenv("AZURE_SWEDEN_API_BASE"), + "api_key": os.getenv("AZURE_SWEDEN_API_KEY"), "metadata": { "model_info": { "base_model": "azure/dall-e-3", diff --git a/tests/litellm_utils_tests/test_supports_tool_choice.py b/tests/litellm_utils_tests/test_supports_tool_choice.py index cfa190f74b1..e09be153195 100644 --- a/tests/litellm_utils_tests/test_supports_tool_choice.py +++ b/tests/litellm_utils_tests/test_supports_tool_choice.py @@ -137,6 +137,8 @@ async def test_supports_tool_choice(): or model_name in block_list or "azure/eu" in model_name or "azure/us" in model_name + or "o1" in model_name + or "o3" in model_name ): continue diff --git a/tests/llm_translation/test_azure_o_series.py b/tests/llm_translation/test_azure_o_series.py index 13ba4169ceb..3d3764f0492 100644 --- a/tests/llm_translation/test_azure_o_series.py +++ b/tests/llm_translation/test_azure_o_series.py @@ -21,9 +21,10 @@ from base_llm_unit_tests import BaseLLMChatTest, BaseOSeriesModelsTest class TestAzureOpenAIO1(BaseOSeriesModelsTest, BaseLLMChatTest): def get_base_completion_call_args(self): return { - "model": "azure/o1-preview", + "model": "azure/o1", "api_key": os.getenv("AZURE_OPENAI_O1_KEY"), - "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com", + "api_base": "https://openai-prod-test.openai.azure.com", + "api_version": "2024-12-01-preview" } def get_client(self): @@ -31,7 +32,7 @@ class TestAzureOpenAIO1(BaseOSeriesModelsTest, BaseLLMChatTest): return AzureOpenAI( api_key="my-fake-o1-key", - base_url="https://openai-gpt-4-test-v-1.openai.azure.com", + base_url="https://openai-prod-test.openai.azure.com", api_version="2024-02-15-preview", ) @@ -170,3 +171,54 @@ def test_openai_o_series_max_retries_0(mock_get_openai_client): mock_get_openai_client.assert_called_once() assert mock_get_openai_client.call_args.kwargs["max_retries"] == 0 + + +@pytest.mark.asyncio +async def test_azure_o1_series_response_format_extra_params(): + """ + Tool calling should work for all azure o_series models. + """ + litellm._turn_on_debug() + + from openai import AsyncAzureOpenAI + + litellm.set_verbose = True + + client = AsyncAzureOpenAI( + api_key="fake-api-key", + base_url="https://openai-prod-test.openai.azure.com/openai/deployments/o1/chat/completions?api-version=2025-01-01-preview", + api_version="2025-01-01-preview" + ) + + tools = [{'type': 'function', 'function': {'name': 'get_current_time', 'description': 'Get the current time in a given location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city name, e.g. San Francisco'}}, 'required': ['location']}}}] + response_format = {'type': 'json_object'} + tool_choice = "auto" + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_client: + try: + await litellm.acompletion( + client=client, + model="azure/o_series/", + api_key="xxxxx", + api_base="https://openai-prod-test.openai.azure.com/openai/deployments/o1/chat/completions?api-version=2025-01-01-preview", + api_version="2024-12-01-preview", + messages=[{"role": "user", "content": "Hello! return a json object"}], + tools=tools, + response_format=response_format, + tool_choice=tool_choice + ) + except Exception as e: + print(f"Error: {e}") + + mock_client.assert_called_once() + request_body = mock_client.call_args.kwargs + + print("request_body: ", json.dumps(request_body, indent=4)) + assert request_body["tools"] == tools + assert request_body["response_format"] == response_format + assert request_body["tool_choice"] == tool_choice + + + + diff --git a/tests/llm_translation/test_cohere.py b/tests/llm_translation/test_cohere.py index 124a5c87888..6b4d3a2045b 100644 --- a/tests/llm_translation/test_cohere.py +++ b/tests/llm_translation/test_cohere.py @@ -17,6 +17,9 @@ import pytest import litellm from litellm import RateLimitError, Timeout, completion, completion_cost, embedding +from unittest.mock import AsyncMock, patch +from litellm import RateLimitError, Timeout, completion, completion_cost, embedding +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler litellm.num_retries = 3 @@ -224,3 +227,57 @@ async def test_chat_completion_cohere_stream(sync_mode): pass except Exception as e: pytest.fail(f"Error occurred: {e}") + + +@pytest.mark.asyncio +async def test_cohere_request_body_with_allowed_params(): + """ + Test to validate that when allowed_openai_params is provided, the request body contains + the correct response_format and reasoning_effort values. + """ + # Define test parameters + test_response_format = {"type": "json"} + test_reasoning_effort = "low" + test_tools = [{ + "type": "function", + "function": { + "name": "get_current_time", + "description": "Get the current time in a given location.", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name, e.g. San Francisco"} + }, + "required": ["location"] + } + } + }] + + client = AsyncHTTPHandler() + + # Mock the post method + with patch.object(client, "post", new=AsyncMock()) as mock_post: + try: + await litellm.acompletion( + model="cohere/command", + messages=[{"content": "what llm are you", "role": "user"}], + allowed_openai_params=["tools", "response_format", "reasoning_effort"], + response_format=test_response_format, + reasoning_effort=test_reasoning_effort, + tools=test_tools, + client=client + ) + except Exception: + pass # We only care about the request body validation + + # Verify the API call was made + mock_post.assert_called_once() + + # Get and parse the request body + request_data = json.loads(mock_post.call_args.kwargs["data"]) + print(f"request_data: {request_data}") + + # Validate request contains our specified parameters + assert "allowed_openai_params" not in request_data + assert request_data["response_format"] == test_response_format + assert request_data["reasoning_effort"] == test_reasoning_effort diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 45dc3d8f6f0..5e792d46e9f 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -67,6 +67,30 @@ def test_anthropic_optional_params(stop_sequence, expected_count): assert len(optional_params) == expected_count + + +def test_get_optional_params_with_allowed_openai_params(): + """ + Test if use can dynamically pass in allowed_openai_params to override default behavior + """ + litellm.drop_params = True + tools = [{'type': 'function', 'function': {'name': 'get_current_time', 'description': 'Get the current time in a given location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city name, e.g. San Francisco'}}, 'required': ['location']}}}] + response_format = {"type": "json"} + reasoning_effort = "low" + optional_params = get_optional_params( + model="cf/llama-3.1-70b-instruct", + custom_llm_provider="cloudflare", + allowed_openai_params=["tools", "reasoning_effort", "response_format"], + tools=tools, + response_format=response_format, + reasoning_effort=reasoning_effort, + ) + print(f"optional_params: {optional_params}") + assert optional_params["tools"] == tools + assert optional_params["response_format"] == response_format + assert optional_params["reasoning_effort"] == reasoning_effort + + def test_bedrock_optional_params_embeddings(): litellm.drop_params = True optional_params = get_optional_params_embeddings( @@ -1380,6 +1404,16 @@ def test_azure_modalities_param(): assert optional_params["modalities"] == ["text", "audio"] assert optional_params["audio"] == {"type": "audio_input", "input": "test.wav"} + + +def test_azure_response_format_param(): + optional_params = litellm.get_optional_params( + model="azure/o_series/test-o3-mini", + custom_llm_provider="azure/o_series", + tools= [{'type': 'function', 'function': {'name': 'get_current_time', 'description': 'Get the current time in a given location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city name, e.g. San Francisco'}}, 'required': ['location']}}}] + ) + + @pytest.mark.parametrize( "model, provider", [ @@ -1396,3 +1430,4 @@ def test_anthropic_unified_reasoning_content(model, provider): reasoning_effort="high", ) assert optional_params["thinking"] == {"type": "enabled", "budget_tokens": 4096} + diff --git a/tests/local_testing/test_bad_params.py b/tests/local_testing/test_bad_params.py index ef3b4596ec0..221135df901 100644 --- a/tests/local_testing/test_bad_params.py +++ b/tests/local_testing/test_bad_params.py @@ -44,7 +44,7 @@ def test_completion_invalid_param_cohere(): except Exception as e: assert isinstance(e, litellm.UnsupportedParamsError) print("got an exception=", str(e)) - if " cohere does not support parameters: {'seed': 12}" in str(e): + if "cohere does not support parameters: ['seed']" in str(e): pass else: pytest.fail(f"An error occurred {e}") diff --git a/tests/local_testing/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py index b0ebcf77673..222572935b3 100644 --- a/tests/local_testing/test_custom_callback_input.py +++ b/tests/local_testing/test_custom_callback_input.py @@ -1133,10 +1133,10 @@ def test_image_generation_openai(): response = litellm.image_generation( prompt="A cute baby sea otter", - model="azure/", - api_base=os.getenv("AZURE_API_BASE"), - api_key=os.getenv("AZURE_API_KEY"), - api_version="2023-06-01-preview", + model="azure/dall-e-3-test", + api_version="2023-12-01-preview", + api_base=os.getenv("AZURE_SWEDEN_API_BASE"), + api_key=os.getenv("AZURE_SWEDEN_API_KEY"), ) print(f"response: {response}") diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index 20a2f28c958..68a79f94a67 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -1136,16 +1136,7 @@ async def test_aimg_gen_on_router(): "api_base": os.getenv("AZURE_SWEDEN_API_BASE"), "api_key": os.getenv("AZURE_SWEDEN_API_KEY"), }, - }, - { - "model_name": "dall-e-2", - "litellm_params": { - "model": "azure/", - "api_version": "2023-06-01-preview", - "api_base": os.getenv("AZURE_API_BASE"), - "api_key": os.getenv("AZURE_API_KEY"), - }, - }, + } ] router = Router(model_list=model_list, num_retries=3) response = await router.aimage_generation( @@ -1153,13 +1144,6 @@ async def test_aimg_gen_on_router(): ) print(response) assert len(response.data) > 0 - - response = await router.aimage_generation( - model="dall-e-2", prompt="A cute baby sea otter" - ) - print(response) - assert len(response.data) > 0 - router.reset() except litellm.InternalServerError as e: pass