From ee9dd158dd51e50f47a0f5b90cb29fc6b6ced745 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Mon, 30 Jun 2025 21:56:03 -0700 Subject: [PATCH] Fix - handle empty config.yaml + Fix gemini /models - replace models/ as expected, instead of using 'strip' (#12189) * fix(proxy_server.py): handle empty config yaml Fixes https://github.com/BerriAI/litellm/issues/12163 * fix(gemini/common_utils.py): replace models/ as expected, instead of using 'strip' Fixes https://github.com/BerriAI/litellm/issues/12160 * fix(anthropic/experimental_pass_through/messages/transformation.py): check for env var when selecting api key * docs(config_settings.md): add api key to docs --- docs/my-website/docs/proxy/config_settings.md | 1 + .../messages/transformation.py | 5 ++ litellm/llms/gemini/common_utils.py | 14 +-- litellm/proxy/_new_secret_config.yaml | 28 ++---- litellm/proxy/proxy_server.py | 2 + .../test_anthropic_completion.py | 25 ++++++ .../llms/gemini/test_gemini_common_utils.py | 86 +++++++++++++++++++ tests/test_litellm/proxy/test_proxy_server.py | 72 ++++++++++++++++ 8 files changed, 205 insertions(+), 28 deletions(-) create mode 100644 tests/test_litellm/llms/gemini/test_gemini_common_utils.py diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 482b599d634..db011a5825c 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -319,6 +319,7 @@ router_settings: | ATHINA_API_KEY | API key for Athina service | ATHINA_BASE_URL | Base URL for Athina service (defaults to `https://log.athina.ai`) | AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key) +| ANTHROPIC_API_KEY | API key for Anthropic service | AWS_ACCESS_KEY_ID | Access Key ID for AWS services | AWS_PROFILE_NAME | AWS CLI profile name to be used | AWS_REGION_NAME | Default AWS region for service interactions diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 160d4eafb57..46ba96f2605 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -60,12 +60,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> Tuple[dict, Optional[str]]: + import os + + if api_key is None: + api_key = os.getenv("ANTHROPIC_API_KEY") if "x-api-key" not in headers and api_key: headers["x-api-key"] = api_key if "anthropic-version" not in headers: headers["anthropic-version"] = DEFAULT_ANTHROPIC_API_VERSION if "content-type" not in headers: headers["content-type"] = "application/json" + return headers, api_base def transform_anthropic_messages_request( diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 3331f584b51..31b57434e10 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -50,6 +50,14 @@ class GeminiModelInfo(BaseLLMModelInfo): def get_base_model(model: str) -> Optional[str]: return model.replace("gemini/", "") + def process_model_name(self, models: List[Dict[str, str]]) -> List[str]: + litellm_model_names = [] + for model in models: + stripped_model_name = model["name"].replace("models/", "") + litellm_model_name = "gemini/" + stripped_model_name + litellm_model_names.append(litellm_model_name) + return litellm_model_names + def get_models( self, api_key: Optional[str] = None, api_base: Optional[str] = None ) -> List[str]: @@ -72,11 +80,7 @@ class GeminiModelInfo(BaseLLMModelInfo): models = response.json()["models"] - litellm_model_names = [] - for model in models: - stripped_model_name = model["name"].strip("models/") - litellm_model_name = "gemini/" + stripped_model_name - litellm_model_names.append(litellm_model_name) + litellm_model_names = self.process_model_name(models) return litellm_model_names def get_error_class( diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 4c84ff99a83..a854922f4ca 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,25 +1,7 @@ model_list: - - model_name: gemini-2.5-pro + - model_name: "gemini/*" litellm_params: - model: gemini/gemini-2.5-pro - - model_name: azure-batches - litellm_params: - model: azure/gpt-4o - api_key: os.environ/AZURE_API_KEY_HIDDEN - api_base: os.environ/AZURE_API_BASE_HIDDEN - - model_name: openai-gpt-4o - litellm_params: - model: gpt-4o - api_key: os.environ/OPENAI_API_KEY_TEST - model_info: - id: 12345678 - - model_name: openai-gpt-4o - litellm_params: - model: gpt-4o - api_key: os.environ/OPENAI_API_KEY_TEST_2 - model_info: - id: 12345679 - - -general_settings: - check_managed_files_batch_cost: true \ No newline at end of file + model: "gemini/*" + api_key: os.environ/GEMINI_API_KEY +litellm_settings: + check_provider_endpoint: true \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a30112bcbde..c59a0ec5e05 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1387,6 +1387,8 @@ class ProxyConfig: "litellm_settings": {}, } + if config is None: + raise Exception("Config cannot be None or Empty.") # Process includes config = self._process_includes( config=config, base_dir=os.path.dirname(os.path.abspath(file_path or "")) diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index b87bd693a94..24c5b656f45 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1342,3 +1342,28 @@ def test_anthropic_prefix_prompt(): print(f"response: {response}") assert response is not None assert response.choices[0].message.content.startswith("Argentina") + + +@pytest.mark.asyncio +async def test_claude_tool_use_with_anthropic_acreate(): + response = await litellm.anthropic.messages.acreate( + messages=[ + {"role": "user", "content": "Hello, can you tell me the weather in Boston?"} + ], + model="anthropic/claude-3-5-sonnet-20240620", + stream=True, + max_tokens=100, + tools=[ + { + "name": "get_weather", + "description": "Get current weather information for a specific location", + "input_schema": { + "type": "object", + "properties": {"location": {"type": "string"}}, + }, + } + ], + ) + + async for chunk in response: + print(chunk) diff --git a/tests/test_litellm/llms/gemini/test_gemini_common_utils.py b/tests/test_litellm/llms/gemini/test_gemini_common_utils.py new file mode 100644 index 00000000000..6aca755506f --- /dev/null +++ b/tests/test_litellm/llms/gemini/test_gemini_common_utils.py @@ -0,0 +1,86 @@ +import pytest + +from litellm.llms.gemini.common_utils import GeminiModelInfo + + +class TestGeminiModelInfo: + """Test suite for GeminiModelInfo class""" + + def test_process_model_name_normal_cases(self): + """Test process_model_name with normal model names""" + gemini_model_info = GeminiModelInfo() + + # Test with normal model names + models = [ + {"name": "models/gemini-1.5-flash"}, + {"name": "models/gemini-1.5-pro"}, + {"name": "models/gemini-2.0-flash-exp"}, + ] + + result = gemini_model_info.process_model_name(models) + + expected = [ + "gemini/gemini-1.5-flash", + "gemini/gemini-1.5-pro", + "gemini/gemini-2.0-flash-exp", + ] + + assert result == expected + + def test_process_model_name_edge_cases(self): + """Test process_model_name with edge cases that could be affected by strip() vs replace()""" + gemini_model_info = GeminiModelInfo() + + # Test edge cases where model names end with characters from "models/" + # These would be incorrectly processed if using strip("models/") instead of replace("models/", "") + models = [ + { + "name": "models/gemini-1.5-pro" + }, # ends with 'o' - would become "gemini-1.5-pr" with strip() + { + "name": "models/test-model" + }, # ends with 'l' - would become "gemini/test-mode" with strip() + { + "name": "models/custom-models" + }, # ends with 's' - would become "gemini/custom-model" with strip() + { + "name": "models/demo" + }, # ends with 'o' - would become "gemini/dem" with strip() + ] + + result = gemini_model_info.process_model_name(models) + + expected = [ + "gemini/gemini-1.5-pro", # 'o' should be preserved + "gemini/test-model", # 'l' should be preserved + "gemini/custom-models", # 's' should be preserved + "gemini/demo", # 'o' should be preserved + ] + + assert result == expected + + def test_process_model_name_empty_list(self): + """Test process_model_name with empty list""" + gemini_model_info = GeminiModelInfo() + + result = gemini_model_info.process_model_name([]) + + assert result == [] + + def test_process_model_name_no_models_prefix(self): + """Test process_model_name with model names that don't have 'models/' prefix""" + gemini_model_info = GeminiModelInfo() + + models = [ + {"name": "gemini-1.5-flash"}, # No "models/" prefix + {"name": "custom-model"}, + ] + + result = gemini_model_info.process_model_name(models) + + expected = [ + "gemini/gemini-1.5-flash", + "gemini/custom-model", + ] + + assert result == expected diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 056228ac0a9..75fffeac5a9 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -539,3 +539,75 @@ async def test_delete_deployment_type_mismatch(): assert ( "12345679" not in deleted_ids ), f"Model 12345679 should NOT be deleted. Deleted IDs: {deleted_ids}" + + +@pytest.mark.asyncio +async def test_get_config_from_file(tmp_path, monkeypatch): + """ + Test the _get_config_from_file method of ProxyConfig class. + Tests various scenarios: valid file, non-existent file, no file path, None config. + """ + import yaml + + from litellm.proxy.proxy_server import ProxyConfig + + # Create a ProxyConfig instance + proxy_config = ProxyConfig() + + # Test Case 1: Valid YAML config file exists + test_config = { + "model_list": [{"model_name": "gpt-4", "litellm_params": {"model": "gpt-4"}}], + "general_settings": {"master_key": "sk-test"}, + "router_settings": {"enable_pre_call_checks": True}, + "litellm_settings": {"drop_params": True}, + } + + config_file = tmp_path / "test_config.yaml" + with open(config_file, "w") as f: + yaml.dump(test_config, f) + + # Clear global user_config_file_path for this test + monkeypatch.setattr("litellm.proxy.proxy_server.user_config_file_path", None) + + result = await proxy_config._get_config_from_file(str(config_file)) + assert result == test_config + + # Verify that user_config_file_path was set + from litellm.proxy.proxy_server import user_config_file_path + + assert user_config_file_path == str(config_file) + + # Test Case 2: File path provided but file doesn't exist + non_existent_file = tmp_path / "non_existent.yaml" + + with pytest.raises(Exception, match=f"Config file not found: {non_existent_file}"): + await proxy_config._get_config_from_file(str(non_existent_file)) + + # Test Case 3: No file path provided (should return default config) + monkeypatch.setattr("litellm.proxy.proxy_server.user_config_file_path", None) + + expected_default = { + "model_list": [], + "general_settings": {}, + "router_settings": {}, + "litellm_settings": {}, + } + + result = await proxy_config._get_config_from_file(None) + assert result == expected_default + + # Test Case 4: Empty YAML file (should raise exception for None config) + empty_file = tmp_path / "empty_config.yaml" + with open(empty_file, "w") as f: + f.write("") # Write empty content which will result in None when loaded + + with pytest.raises(Exception, match="Config cannot be None or Empty."): + await proxy_config._get_config_from_file(str(empty_file)) + + # Test Case 5: Using global user_config_file_path when no config_file_path provided + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_config_file_path", str(config_file) + ) + + result = await proxy_config._get_config_from_file(None) + assert result == test_config