From 498c4254ead47876814b8a4f9b2fce485b943a37 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sun, 7 Dec 2025 20:53:51 +0800 Subject: [PATCH 001/699] fix: Return 403 exception when calling GET responses api --- litellm/proxy/auth/auth_checks.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index fc79a4d3591..309bd577606 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -402,13 +402,14 @@ def _allowed_routes_check(user_route: str, allowed_routes: list) -> bool: - user_route: str - the route the user is trying to call - allowed_routes: List[str|LiteLLMRoutes] - the list of allowed routes for the user. """ + from starlette.routing import compile_path for allowed_route in allowed_routes: - if ( - allowed_route in LiteLLMRoutes.__members__ - and user_route in LiteLLMRoutes[allowed_route].value - ): - return True + if allowed_route in LiteLLMRoutes.__members__: + for template in LiteLLMRoutes[allowed_route].value: + regex, _, _ = compile_path(template) + if regex.match(user_route): + return True elif allowed_route == user_route: return True return False From 4ab58619ad56d93eb4add67bfd6064c50f449fa1 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sun, 14 Dec 2025 17:55:13 +0800 Subject: [PATCH 002/699] fix: added new step into rotate master key function for processing credentials table --- .../proxy/credential_endpoints/endpoints.py | 9 ++--- .../key_management_endpoints.py | 34 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 647abb73648..9f228bb1184 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -21,11 +21,11 @@ router = APIRouter() class CredentialHelperUtils: @staticmethod - def encrypt_credential_values(credential: CredentialItem) -> CredentialItem: + def encrypt_credential_values(credential: CredentialItem, new_encryption_key: Optional[str] = None) -> CredentialItem: """Encrypt values in credential.credential_values and add to DB""" encrypted_credential_values = {} for key, value in (credential.credential_values or {}).items(): - encrypted_credential_values[key] = encrypt_value_helper(value) + encrypted_credential_values[key] = encrypt_value_helper(value, new_encryption_key) # Return a new object to avoid mutating the caller's credential, which # is kept in memory and should remain unencrypted. @@ -246,7 +246,7 @@ async def delete_credential( def update_db_credential( - db_credential: CredentialItem, updated_patch: CredentialItem + db_credential: CredentialItem, updated_patch: CredentialItem, new_encryption_key: Optional[str] = None ) -> CredentialItem: """ Update a credential in the DB. @@ -258,7 +258,8 @@ def update_db_credential( ) encrypted_credential = CredentialHelperUtils.encrypt_credential_values( - updated_patch + updated_patch, + new_encryption_key, ) # update model name if encrypted_credential.credential_name: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index da44bda791d..8ea3122ce01 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2539,6 +2539,40 @@ async def _rotate_master_key( new_master_key=new_master_key, ) + # 5. process credentials table + try: + credentials = await prisma_client.db.litellm_credentialstable.find_many() + except Exception: + credentials = None + if credentials: + from litellm.proxy.credential_endpoints.endpoints import update_db_credential + + for cred in credentials: + try: + decrypted_cred = proxy_config.decrypt_credentials(cred) + encrypted_cred = update_db_credential( + db_credential=cred, + updated_patch=decrypted_cred, + new_encryption_key=new_master_key, + ) + credential_object_jsonified = jsonify_object(encrypted_cred.model_dump()) + await prisma_client.db.litellm_credentialstable.update( + where={"credential_name": cred.credential_name}, + data={ + **credential_object_jsonified, + "updated_by": user_api_key_dict.user_id, + }, + ) + except Exception as e: + verbose_proxy_logger.error( + f"Failed to re-encrypt credential {cred.credential_name}: {str(e)}" + ) + # Continue with next credential instead of failing entire rotation + continue + verbose_proxy_logger.debug( + f"Successfully re-encrypted {len(credentials)} credentials with new master key" + ) + def get_new_token(data: Optional[RegenerateKeyRequest]) -> str: if data and data.new_key is not None: From 7b6a00d3df82352ae1cdb76f6e7a5c6ac632b7e6 Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 16 Dec 2025 13:06:33 -0300 Subject: [PATCH 003/699] fix: remove deprecated Groq models and update model registry - Remove 20 deprecated/unavailable Groq models from registry - Add groq/meta-llama/llama-guard-4-12b (new safety model) - Add supports_vision to Llama 4 models (maverick, scout) - Update Groq documentation with current model list - Clean up test file references to deprecated models Fixes #18043 --- docs/my-website/docs/providers/groq.md | 31 +-- ...odel_prices_and_context_window_backup.json | 240 +----------------- model_prices_and_context_window.json | 240 +----------------- tests/test_litellm/test_utils.py | 2 - 4 files changed, 24 insertions(+), 489 deletions(-) diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md index ebed31f720f..55c222635d2 100644 --- a/docs/my-website/docs/providers/groq.md +++ b/docs/my-website/docs/providers/groq.md @@ -150,15 +150,15 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion | Model Name | Usage | |--------------------|---------------------------------------------------------| -| llama-3.1-8b-instant | `completion(model="groq/llama-3.1-8b-instant", messages)` | -| llama-3.1-70b-versatile | `completion(model="groq/llama-3.1-70b-versatile", messages)` | -| llama3-8b-8192 | `completion(model="groq/llama3-8b-8192", messages)` | -| llama3-70b-8192 | `completion(model="groq/llama3-70b-8192", messages)` | -| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` | -| mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` | -| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` | -| moonshotai/kimi-k2-instruct | `completion(model="groq/moonshotai/kimi-k2-instruct", messages)` | -| qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` | +| llama-3.3-70b-versatile | `completion(model="groq/llama-3.3-70b-versatile", messages)` | +| llama-3.1-8b-instant | `completion(model="groq/llama-3.1-8b-instant", messages)` | +| meta-llama/llama-4-scout-17b-16e-instruct | `completion(model="groq/meta-llama/llama-4-scout-17b-16e-instruct", messages)` | +| meta-llama/llama-4-maverick-17b-128e-instruct | `completion(model="groq/meta-llama/llama-4-maverick-17b-128e-instruct", messages)` | +| meta-llama/llama-guard-4-12b | `completion(model="groq/meta-llama/llama-guard-4-12b", messages)` | +| qwen/qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` | +| moonshotai/kimi-k2-instruct-0905 | `completion(model="groq/moonshotai/kimi-k2-instruct-0905", messages)` | +| openai/gpt-oss-120b | `completion(model="groq/openai/gpt-oss-120b", messages)` | +| openai/gpt-oss-20b | `completion(model="groq/openai/gpt-oss-20b", messages)` | ## Groq - Tool / Function Calling Example @@ -261,31 +261,28 @@ if tool_calls: print("second response\n", second_response) ``` -## Groq - Vision Example +## Groq - Vision Example -Select Groq models support vision. Check out their [model list](https://console.groq.com/docs/vision) for more details. +Groq's Llama 4 models support vision. Check out their [model list](https://console.groq.com/docs/vision) for more details. ```python -from litellm import completion - -import os +import os from litellm import completion os.environ["GROQ_API_KEY"] = "your-api-key" -# openai call response = completion( - model = "groq/llama-3.2-11b-vision-preview", + model = "groq/meta-llama/llama-4-scout-17b-16e-instruct", messages=[ { "role": "user", "content": [ { "type": "text", - "text": "What’s in this image?" + "text": "What's in this image?" }, { "type": "image_url", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2af548ce07e..5754899846c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -17509,75 +17509,6 @@ "supports_response_schema": true, "supports_vision": true }, - "groq/deepseek-r1-distill-llama-70b": { - "input_cost_per_token": 7.5e-07, - "litellm_provider": "groq", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 9.9e-07, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/distil-whisper-large-v3-en": { - "input_cost_per_second": 5.56e-06, - "litellm_provider": "groq", - "mode": "audio_transcription", - "output_cost_per_second": 0.0 - }, - "groq/gemma-7b-it": { - "deprecation_date": "2024-12-18", - "input_cost_per_token": 7e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/gemma2-9b-it": { - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_function_calling": false, - "supports_response_schema": false, - "supports_tool_choice": false - }, - "groq/llama-3.1-405b-reasoning": { - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.1-70b-versatile": { - "deprecation_date": "2025-01-24", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/llama-3.1-8b-instant": { "input_cost_per_token": 5e-08, "litellm_provider": "groq", @@ -17590,97 +17521,6 @@ "supports_response_schema": false, "supports_tool_choice": true }, - "groq/llama-3.2-11b-text-preview": { - "deprecation_date": "2024-10-28", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-11b-vision-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/llama-3.2-1b-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 4e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-3b-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 6e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-90b-text-preview": { - "deprecation_date": "2024-11-25", - "input_cost_per_token": 9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-90b-vision-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/llama-3.3-70b-specdec": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9.9e-07, - "supports_tool_choice": true - }, "groq/llama-3.3-70b-versatile": { "input_cost_per_token": 5.9e-07, "litellm_provider": "groq", @@ -17693,7 +17533,7 @@ "supports_response_schema": false, "supports_tool_choice": true }, - "groq/llama-guard-3-8b": { + "groq/meta-llama/llama-guard-4-12b": { "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, @@ -17702,44 +17542,6 @@ "mode": "chat", "output_cost_per_token": 2e-07 }, - "groq/llama2-70b-4096": { - "input_cost_per_token": 7e-07, - "litellm_provider": "groq", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama3-groq-70b-8192-tool-use-preview": { - "deprecation_date": "2025-01-06", - "input_cost_per_token": 8.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama3-groq-8b-8192-tool-use-preview": { - "deprecation_date": "2025-01-06", - "input_cost_per_token": 1.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "groq", @@ -17750,7 +17552,8 @@ "output_cost_per_token": 6e-07, "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "groq/meta-llama/llama-4-scout-17b-16e-instruct": { "input_cost_per_token": 1.1e-07, @@ -17762,41 +17565,8 @@ "output_cost_per_token": 3.4e-07, "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true - }, - "groq/mistral-saba-24b": { - "input_cost_per_token": 7.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 32000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.9e-07 - }, - "groq/mixtral-8x7b-32768": { - "deprecation_date": "2025-03-20", - "input_cost_per_token": 2.4e-07, - "litellm_provider": "groq", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 2.4e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/moonshotai/kimi-k2-instruct": { - "input_cost_per_token": 1e-06, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 16384, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 3e-06, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "groq/moonshotai/kimi-k2-instruct-0905": { "input_cost_per_token": 1e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2af548ce07e..5754899846c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -17509,75 +17509,6 @@ "supports_response_schema": true, "supports_vision": true }, - "groq/deepseek-r1-distill-llama-70b": { - "input_cost_per_token": 7.5e-07, - "litellm_provider": "groq", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 9.9e-07, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/distil-whisper-large-v3-en": { - "input_cost_per_second": 5.56e-06, - "litellm_provider": "groq", - "mode": "audio_transcription", - "output_cost_per_second": 0.0 - }, - "groq/gemma-7b-it": { - "deprecation_date": "2024-12-18", - "input_cost_per_token": 7e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/gemma2-9b-it": { - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_function_calling": false, - "supports_response_schema": false, - "supports_tool_choice": false - }, - "groq/llama-3.1-405b-reasoning": { - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.1-70b-versatile": { - "deprecation_date": "2025-01-24", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/llama-3.1-8b-instant": { "input_cost_per_token": 5e-08, "litellm_provider": "groq", @@ -17590,97 +17521,6 @@ "supports_response_schema": false, "supports_tool_choice": true }, - "groq/llama-3.2-11b-text-preview": { - "deprecation_date": "2024-10-28", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-11b-vision-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/llama-3.2-1b-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 4e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-3b-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 6e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-90b-text-preview": { - "deprecation_date": "2024-11-25", - "input_cost_per_token": 9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-90b-vision-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/llama-3.3-70b-specdec": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9.9e-07, - "supports_tool_choice": true - }, "groq/llama-3.3-70b-versatile": { "input_cost_per_token": 5.9e-07, "litellm_provider": "groq", @@ -17693,7 +17533,7 @@ "supports_response_schema": false, "supports_tool_choice": true }, - "groq/llama-guard-3-8b": { + "groq/meta-llama/llama-guard-4-12b": { "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, @@ -17702,44 +17542,6 @@ "mode": "chat", "output_cost_per_token": 2e-07 }, - "groq/llama2-70b-4096": { - "input_cost_per_token": 7e-07, - "litellm_provider": "groq", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama3-groq-70b-8192-tool-use-preview": { - "deprecation_date": "2025-01-06", - "input_cost_per_token": 8.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama3-groq-8b-8192-tool-use-preview": { - "deprecation_date": "2025-01-06", - "input_cost_per_token": 1.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "groq", @@ -17750,7 +17552,8 @@ "output_cost_per_token": 6e-07, "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "groq/meta-llama/llama-4-scout-17b-16e-instruct": { "input_cost_per_token": 1.1e-07, @@ -17762,41 +17565,8 @@ "output_cost_per_token": 3.4e-07, "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true - }, - "groq/mistral-saba-24b": { - "input_cost_per_token": 7.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 32000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.9e-07 - }, - "groq/mixtral-8x7b-32768": { - "deprecation_date": "2025-03-20", - "input_cost_per_token": 2.4e-07, - "litellm_provider": "groq", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 2.4e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/moonshotai/kimi-k2-instruct": { - "input_cost_per_token": 1e-06, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 16384, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 3e-06, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "groq/moonshotai/kimi-k2-instruct-0905": { "input_cost_per_token": 1e-06, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 2bd94488ba2..a3dc6f5085d 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -870,8 +870,6 @@ SKIP_MODELS = [ "jamba", "deepinfra", "mistral.", - "groq/llama-guard-3-8b", - "groq/gemma2-9b-it", ] # Bedrock models to block - organized by type From 9274860aa28c5d69d696077b92283dfcad90e0a5 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 14:54:26 -0800 Subject: [PATCH 004/699] Base commit --- litellm/proxy/management_endpoints/ui_sso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index d1db21a2706..5094fc5de97 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -594,7 +594,7 @@ def _build_sso_user_update_data( user_id: Optional[str], ) -> dict: """ - Build the update data dictionary for SSO user upsert. + Build the update data dictionary for SSO user upsert Args: result: The SSO response containing user information From 42d7d757a3bb1e5da02ed53c494ce6abca2f7e61 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 15:48:26 -0800 Subject: [PATCH 005/699] Adding role mappings to SSOConfig DB --- litellm/proxy/management_endpoints/ui_sso.py | 2 +- litellm/proxy/proxy_server.py | 1 + .../proxy_setting_endpoints.py | 12 ++ .../proxy/management_endpoints/ui_sso.py | 34 ++++- .../test_proxy_setting_endpoints.py | 135 ++++++++++++++++++ 5 files changed, 182 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 5094fc5de97..d1db21a2706 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -594,7 +594,7 @@ def _build_sso_user_update_data( user_id: Optional[str], ) -> dict: """ - Build the update data dictionary for SSO user upsert + Build the update data dictionary for SSO user upsert. Args: result: The SSO response containing user information diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8fdea95d7ad..dfadab1d531 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3578,6 +3578,7 @@ class ProxyConfig: ) if sso_settings is not None: # Capitalize all keys in sso_settings dictionary + sso_settings.sso_settings.pop("role_mappings", None) uppercase_sso_settings = { key.upper(): value for key, value in sso_settings.sso_settings.items() diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 9c99b625e9f..d9a41d38b22 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -433,10 +433,21 @@ async def get_sso_settings(): if sso_db_record and sso_db_record.sso_settings: # Load settings from database sso_settings_dict = dict(sso_db_record.sso_settings) + + # Extract role_mappings before removing it (it's a dict, not an env variable) + role_mappings_data = sso_settings_dict.pop("role_mappings", None) + role_mappings = None + if role_mappings_data: + from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings + if isinstance(role_mappings_data, dict): + role_mappings = RoleMappings(**role_mappings_data) + elif isinstance(role_mappings_data, RoleMappings): + role_mappings = role_mappings_data decrypted_sso_settings_dict = proxy_config._decrypt_and_set_db_env_variables(environment_variables=sso_settings_dict) # Build SSO config with database values or environment fallback + sso_config = SSOConfig( google_client_id=decrypted_sso_settings_dict.get("google_client_id", None), google_client_secret=decrypted_sso_settings_dict.get("google_client_secret", None), @@ -451,6 +462,7 @@ async def get_sso_settings(): proxy_base_url=decrypted_sso_settings_dict.get("proxy_base_url", None), user_email=decrypted_sso_settings_dict.get("user_email"), ui_access_mode=decrypted_sso_settings_dict.get("ui_access_mode"), + role_mappings=role_mappings, ) # Get the schema for UI display diff --git a/litellm/types/proxy/management_endpoints/ui_sso.py b/litellm/types/proxy/management_endpoints/ui_sso.py index 820b0164400..187d8c97c05 100644 --- a/litellm/types/proxy/management_endpoints/ui_sso.py +++ b/litellm/types/proxy/management_endpoints/ui_sso.py @@ -1,10 +1,12 @@ -from typing import List, Literal, Optional, Union +from typing import Dict, List, Literal, Optional, Union from pydantic import Field from typing_extensions import TypedDict from litellm.types.utils import LiteLLMPydanticObjectBase +from litellm.proxy._types import LitellmUserRoles + class LiteLLM_UpperboundKeyGenerateParams(LiteLLMPydanticObjectBase): """ @@ -60,6 +62,30 @@ class AccessControl_UI_AccessMode(LiteLLMPydanticObjectBase): sso_group_jwt_field: str +class RoleMappings(LiteLLMPydanticObjectBase): + """ + Configuration for mapping SSO groups to LiteLLM roles. + + The system will look at the group_claim field in the SSO token to determine + which role to assign the user based on the roles mapping. + """ + + provider: str = Field( + description="SSO Provider name (e.g., 'google', 'microsoft', 'generic')" + ) + group_claim: str = Field( + description="The field name in the SSO token that contains the groups array (e.g., 'groups', 'roles')" + ) + default_role: Optional[LitellmUserRoles] = Field( + default=None, + description="Default role to assign if user's groups don't match any role mappings. Must be a valid LitellmUserRoles value (e.g., 'proxy_admin', 'internal_user', 'proxy_admin_viewer')" + ) + roles: Dict[LitellmUserRoles, List[str]] = Field( + default_factory=dict, + description="Mapping of LiteLLM role names to arrays of SSO group names. Example: {'proxy_admin': ['group-1', 'group-2'], 'proxy_admin_viewer': ['group-3']}" + ) + + class SSOConfig(LiteLLMPydanticObjectBase): """ Configuration for SSO environment variables and settings @@ -127,6 +153,12 @@ class SSOConfig(LiteLLMPydanticObjectBase): description="Access mode for the UI", ) + # Role Mappings + role_mappings: Optional[RoleMappings] = Field( + default=None, + description="Configuration for mapping SSO groups to LiteLLM roles based on group claims in the SSO token", + ) + class DefaultTeamSSOParams(LiteLLMPydanticObjectBase): """ diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index d3c99151195..8fdfd6897a8 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -290,6 +290,10 @@ class TestProxySettingEndpoints: assert "google_client_id" in data["field_schema"]["properties"] assert "description" in data["field_schema"]["properties"]["google_client_id"] + # Verify role_mappings is present in response (can be None if not set) + assert "role_mappings" in values + assert values["role_mappings"] is None + # Verify find_unique was called with correct parameters mock_prisma.db.litellm_ssoconfig.find_unique.assert_called_once() call_args = mock_prisma.db.litellm_ssoconfig.find_unique.call_args @@ -863,6 +867,10 @@ class TestProxySettingEndpoints: assert values["google_client_secret"] == "decrypted_google_secret" assert values["microsoft_client_id"] == "decrypted_microsoft_id" assert values["proxy_base_url"] == "https://decrypted.example.com" + + # Verify role_mappings is present in response (can be None if not set) + assert "role_mappings" in values + assert values["role_mappings"] is None def test_update_sso_settings_to_database(self, mock_proxy_config, mock_auth, monkeypatch): """Test updating SSO settings saves to the dedicated database table""" @@ -1062,6 +1070,7 @@ class TestProxySettingEndpoints: assert values.get("google_client_id") is None assert values.get("google_client_secret") is None assert values.get("microsoft_client_id") is None + assert values.get("role_mappings") is None def test_update_sso_settings_no_database_connection(self, mock_proxy_config, mock_auth, monkeypatch): """Test updating SSO settings when database is not connected""" @@ -1088,3 +1097,129 @@ class TestProxySettingEndpoints: data = response.json() assert "error" in data["detail"] assert "Database not connected" in data["detail"]["error"] + + def test_get_sso_settings_with_role_mappings(self, mock_proxy_config, mock_auth, monkeypatch): + """Test getting SSO settings when role_mappings is present in database""" + from unittest.mock import AsyncMock, MagicMock + from litellm.proxy._types import LitellmUserRoles + + # Mock the prisma client with database record containing role_mappings + mock_prisma = MagicMock() + mock_db_record = MagicMock() + mock_db_record.sso_settings = { + "google_client_id": "test_google_client_id", + "role_mappings": { + "provider": "google", + "group_claim": "groups", + "default_role": LitellmUserRoles.INTERNAL_USER, + "roles": { + LitellmUserRoles.PROXY_ADMIN: ["admin-group"], + }, + }, + } + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=mock_db_record) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + # Mock decryption to return the values as-is (role_mappings should not be passed to decryption) + from litellm.proxy.proxy_server import proxy_config + def mock_decrypt(environment_variables): + # role_mappings should not be in environment_variables since it's extracted before decryption + assert "role_mappings" not in environment_variables + return environment_variables + + monkeypatch.setattr( + proxy_config, "_decrypt_and_set_db_env_variables", mock_decrypt + ) + + response = client.get("/get/sso_settings") + + assert response.status_code == 200 + data = response.json() + + # Verify role_mappings is returned correctly + values = data["values"] + assert "role_mappings" in values + assert values["role_mappings"] is not None + assert values["role_mappings"]["provider"] == "google" + assert values["role_mappings"]["group_claim"] == "groups" + assert values["role_mappings"]["default_role"] == LitellmUserRoles.INTERNAL_USER + assert values["role_mappings"]["roles"][LitellmUserRoles.PROXY_ADMIN] == ["admin-group"] + + def test_role_mappings_stored_and_retrieved(self, mock_proxy_config, mock_auth, monkeypatch): + """Test that role_mappings is properly stored and retrieved from SSO settings""" + import json + from unittest.mock import AsyncMock, MagicMock + from litellm.proxy._types import LitellmUserRoles + + monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key") + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + + # Mock the prisma client + mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() + mock_prisma.db.litellm_config = MagicMock() + mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + mock_prisma.db.litellm_config.update = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + # Mock encryption to return values as-is + from litellm.proxy.proxy_server import proxy_config + monkeypatch.setattr(proxy_config, "_encrypt_env_variables", lambda environment_variables: environment_variables) + + # SSO settings with role_mappings + role_mappings_data = { + "provider": "google", + "group_claim": "groups", + "default_role": LitellmUserRoles.INTERNAL_USER, + "roles": { + LitellmUserRoles.PROXY_ADMIN: ["admin-group"], + LitellmUserRoles.INTERNAL_USER: ["user-group"], + }, + } + + new_sso_settings = { + "google_client_id": "test_google_id", + "role_mappings": role_mappings_data, + } + + response = client.patch("/update/sso_settings", json=new_sso_settings) + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert "role_mappings" in data["settings"] + + # Verify role_mappings structure in response + returned_role_mappings = data["settings"]["role_mappings"] + assert returned_role_mappings["provider"] == "google" + assert returned_role_mappings["group_claim"] == "groups" + assert returned_role_mappings["default_role"] == LitellmUserRoles.INTERNAL_USER + assert returned_role_mappings["roles"][LitellmUserRoles.PROXY_ADMIN] == ["admin-group"] + + # Verify upsert was called with role_mappings in the data + assert mock_prisma.db.litellm_ssoconfig.upsert.called + call_args = mock_prisma.db.litellm_ssoconfig.upsert.call_args + create_data = call_args.kwargs["data"]["create"] + stored_sso_settings = json.loads(create_data["sso_settings"]) + assert "role_mappings" in stored_sso_settings + assert stored_sso_settings["role_mappings"]["provider"] == "google" + + # Now test retrieving role_mappings + mock_db_record = MagicMock() + mock_db_record.sso_settings = stored_sso_settings + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=mock_db_record) + monkeypatch.setattr( + proxy_config, "_decrypt_and_set_db_env_variables", lambda environment_variables: environment_variables + ) + + get_response = client.get("/get/sso_settings") + assert get_response.status_code == 200 + get_data = get_response.json() + + # Verify role_mappings is returned correctly + assert "role_mappings" in get_data["values"] + retrieved_role_mappings = get_data["values"]["role_mappings"] + assert retrieved_role_mappings is not None + assert retrieved_role_mappings["provider"] == "google" + assert retrieved_role_mappings["group_claim"] == "groups" + assert retrieved_role_mappings["default_role"] == LitellmUserRoles.INTERNAL_USER From 58330f852d82bd42f02f2e920cdb14ae0b927024 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 17:36:08 -0800 Subject: [PATCH 006/699] WIP waiting for okta --- litellm/proxy/management_endpoints/ui_sso.py | 134 ++++++++++++++++++- 1 file changed, 127 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index d1db21a2706..7a404bcda81 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -85,6 +85,58 @@ else: router = APIRouter() +def determine_role_from_groups( + user_groups: List[str], + role_mappings: "RoleMappings", +) -> Optional[LitellmUserRoles]: + """ + Determine the highest privilege role for a user based on their groups. + + Role hierarchy (highest to lowest): + - proxy_admin + - proxy_admin_viewer + - internal_user + - internal_user_viewer + + Args: + user_groups: List of group names from the SSO token + role_mappings: RoleMappings configuration object + + Returns: + The highest privilege role found, or default_role if no matches, or None + """ + if not role_mappings.roles: + # No role mappings configured, return default_role + return role_mappings.default_role + + # Role hierarchy (highest to lowest) + role_hierarchy = [ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] + + # Convert user_groups to a set for efficient lookup + user_groups_set = set(user_groups) if isinstance(user_groups, list) else set() + + # Find the highest privilege role the user belongs to + for role in role_hierarchy: + if role in role_mappings.roles: + role_groups = role_mappings.roles[role] + if isinstance(role_groups, list) and user_groups_set.intersection(set(role_groups)): + verbose_proxy_logger.debug( + f"User groups {user_groups} matched role '{role.value}' via groups: {role_groups}" + ) + return role + + # No matching groups found, return default_role + verbose_proxy_logger.debug( + f"User groups {user_groups} did not match any role mappings, using default_role: {role_mappings.default_role}" + ) + return role_mappings.default_role + + def process_sso_jwt_access_token( access_token_str: Optional[str], sso_jwt_handler: Optional[JWTHandler], @@ -243,6 +295,7 @@ def generic_response_convertor( response, jwt_handler: JWTHandler, sso_jwt_handler: Optional[JWTHandler] = None, + role_mappings: Optional["RoleMappings"] = None, ) -> CustomOpenID: generic_user_id_attribute_name = os.getenv( "GENERIC_USER_ID_ATTRIBUTE", "preferred_username" @@ -281,16 +334,48 @@ def generic_response_convertor( team_ids = jwt_handler.get_team_ids_from_jwt(cast(dict, response)) all_teams.extend(team_ids) - # Extract user role from SSO response - user_role_from_sso = get_nested_value(response, generic_user_role_attribute_name) + # Determine user role based on role_mappings if available + # Only apply role_mappings for GENERIC SSO provider user_role: Optional[LitellmUserRoles] = None - if user_role_from_sso is not None: - role = get_litellm_user_role(user_role_from_sso) - if role is not None: - user_role = role + + if role_mappings is not None and role_mappings.provider.lower() in ["generic", "okta"]: + # Use role_mappings to determine role from groups + group_claim = role_mappings.group_claim + user_groups_raw = get_nested_value(response, group_claim) + + # Handle different formats: could be a list, string (comma-separated), or single value + user_groups: List[str] = [] + if isinstance(user_groups_raw, list): + user_groups = [str(g) for g in user_groups_raw] + elif isinstance(user_groups_raw, str): + # Handle comma-separated string + user_groups = [g.strip() for g in user_groups_raw.split(",") if g.strip()] + elif user_groups_raw is not None: + # Single value + user_groups = [str(user_groups_raw)] + + if user_groups: + user_role = determine_role_from_groups(user_groups, role_mappings) verbose_proxy_logger.debug( - f"Found valid LitellmUserRoles '{role.value}' from SSO attribute '{generic_user_role_attribute_name}'" + f"Determined role '{user_role.value if user_role else None}' from groups '{user_groups}' using role_mappings" ) + else: + # No groups found, use default_role + user_role = role_mappings.default_role + verbose_proxy_logger.debug( + f"No groups found in '{group_claim}', using default_role: {role_mappings.default_role}" + ) + + # Fallback to existing logic if role_mappings not used + if user_role is None: + user_role_from_sso = get_nested_value(response, generic_user_role_attribute_name) + if user_role_from_sso is not None: + role = get_litellm_user_role(user_role_from_sso) + if role is not None: + user_role = role + verbose_proxy_logger.debug( + f"Found valid LitellmUserRoles '{role.value}' from SSO attribute '{generic_user_role_attribute_name}'" + ) return CustomOpenID( id=get_nested_value(response, generic_user_id_attribute_name), @@ -369,6 +454,40 @@ async def get_generic_sso_response( userinfo_endpoint=generic_userinfo_endpoint, ) + # Get role_mappings from SSO settings if available + role_mappings: Optional["RoleMappings"] = None + try: + from litellm.proxy.utils import get_prisma_client_or_throw + + prisma_client = get_prisma_client_or_throw( + "Prisma client is None, connect a database to your proxy" + ) + + # Get SSO config from dedicated table + sso_db_record = await prisma_client.db.litellm_ssoconfig.find_unique( + where={"id": "sso_config"} + ) + + if sso_db_record and sso_db_record.sso_settings: + sso_settings_dict = dict(sso_db_record.sso_settings) + role_mappings_data = sso_settings_dict.get("role_mappings") + + if role_mappings_data: + from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings + if isinstance(role_mappings_data, dict): + role_mappings = RoleMappings(**role_mappings_data) + elif isinstance(role_mappings_data, RoleMappings): + role_mappings = role_mappings_data + + verbose_proxy_logger.debug( + f"Loaded role_mappings for provider '{role_mappings.provider}'" + ) + except Exception as e: + # If we can't load role_mappings, continue with existing logic + verbose_proxy_logger.debug( + f"Could not load role_mappings from database: {e}. Continuing with existing role logic." + ) + def response_convertor(response, client): nonlocal received_response # return for user debugging received_response = response @@ -376,6 +495,7 @@ async def get_generic_sso_response( response=response, jwt_handler=jwt_handler, sso_jwt_handler=sso_jwt_handler, + role_mappings=role_mappings, ) SSOProvider = create_provider( From f4f5ea85dfe7eff7130724b1d7331354468f5ce8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 18 Dec 2025 14:42:41 +0530 Subject: [PATCH 007/699] Add redisvl in requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c36f94f0752..2fb6c52cfd3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,8 @@ starlette==0.49.1 # starlette fastapi dep backoff==2.2.1 # server dep pyyaml==6.0.2 # server dep uvicorn==0.31.1 # server dep -gunicorn==23.0.0 # server dep +gunicorn==23.0.0 # server depredisvl +redisvl==0.4.1 # redis semantic cache fastuuid==0.13.5 # for uuid4 uvloop==0.21.0 # uvicorn dep, gives us much better performance under load boto3==1.36.0 # aws bedrock/sagemaker calls From 5705aaebbc3247cb1666dec0288ab7b5507b2c79 Mon Sep 17 00:00:00 2001 From: Rens Date: Thu, 18 Dec 2025 15:11:58 +0200 Subject: [PATCH 008/699] Fix Gemini 3 imgs in tool response --- .../prompt_templates/factory.py | 7 +-- ...llm_core_utils_prompt_templates_factory.py | 51 +++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 652692c7b8d..9afea83ef7b 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1496,9 +1496,10 @@ def convert_to_gemini_tool_call_result( content_type = content.get("type", "") if content_type == "text": content_str += content.get("text", "") - elif content_type == "input_image": - # Extract image for inline_data (for Computer Use screenshots) - image_url = content.get("image_url", "") + elif content_type in ("input_image", "image_url"): + # Extract image for inline_data (for Computer Use screenshots and tool results) + image_url_data = content.get("image_url", "") + image_url = image_url_data.get("url", "") if isinstance(image_url_data, dict) else image_url_data if image_url: # Convert image to base64 blob format for Gemini diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 41ac893b4d7..c8fe6efeaa1 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -497,6 +497,57 @@ def test_convert_gemini_messages(): ) +def test_convert_gemini_tool_call_result_with_image_url(): + """ + Test that image_url content type in tool results is handled correctly for Gemini. + Fixes: https://github.com/BerriAI/litellm/issues/18187 + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_result, + ) + from litellm.types.llms.openai import ChatCompletionToolMessage + + # Test with string image_url format + message_str_format = ChatCompletionToolMessage( + role="tool", + tool_call_id="call_123", + content=[{"type": "image_url", "image_url": "data:image/jpeg;base64,/9j/4AAQ"}], + ) + last_message_with_tool_calls = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "index": 0, + "function": {"name": "get_image", "arguments": "{}"}, + } + ], + } + + result = convert_to_gemini_tool_call_result( + message=message_str_format, + last_message_with_tool_calls=last_message_with_tool_calls, + ) + # Should have inline_data for the image + assert isinstance(result, list) and any("inline_data" in p for p in result) + + # Test with dict image_url format (OpenAI standard) + message_dict_format = ChatCompletionToolMessage( + role="tool", + tool_call_id="call_456", + content=[{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"}}], + ) + last_message_with_tool_calls["tool_calls"][0]["id"] = "call_456" + + result2 = convert_to_gemini_tool_call_result( + message=message_dict_format, + last_message_with_tool_calls=last_message_with_tool_calls, + ) + assert isinstance(result2, list) and any("inline_data" in p for p in result2) + + def test_bedrock_tools_unpack_defs(): """ Test that the unpack_defs method handles nested $ref inside anyOf items correctly From e08b4767e925c5c55c62a7881440da4bd6641a7c Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 18 Dec 2025 17:25:43 -0300 Subject: [PATCH 009/699] fix: case-insensitive model cost map lookup Users were getting "does not support parameters: ['tools']" errors when using lowercase model names (e.g., "qwen/qwen3-next-80b-a3b-thinking") because the model cost map has mixed-case keys and the lookup was case-sensitive. Added _get_model_cost_key() helper that tries exact match first (O(1)), then falls back to case-insensitive search if not found. --- litellm/utils.py | 96 ++++++++++++++-------- tests/local_testing/test_get_model_info.py | 83 +++++++++++++++++++ 2 files changed, 144 insertions(+), 35 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index dfc0df5c9a2..8eb71817111 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4697,6 +4697,25 @@ def _strip_model_name(model: str, custom_llm_provider: Optional[str]) -> str: return model +def _get_model_cost_key(potential_key: str) -> Optional[str]: + """ + Get the actual key from model_cost, with case-insensitive fallback. + + Returns the key if found (exact match preferred, then case-insensitive), or None if not found. + """ + # Try exact match first (most common case, O(1)) + if potential_key in litellm.model_cost: + return potential_key + + # Fallback to case-insensitive match + potential_key_lower = potential_key.lower() + for key in litellm.model_cost: + if key.lower() == potential_key_lower: + return key + + return None + + def _get_model_info_from_model_cost(key: str) -> dict: return litellm.model_cost[key] @@ -4845,10 +4864,10 @@ def _is_potential_model_name_in_model_cost( potential_model_names: PotentialModelNamesAndCustomLLMProvider, ) -> bool: """ - Check if the potential model name is in the model cost. + Check if the potential model name is in the model cost (case-insensitive). """ return any( - potential_model_name in litellm.model_cost + _get_model_cost_key(potential_model_name) is not None for potential_model_name in potential_model_names.values() ) @@ -4926,44 +4945,51 @@ def _get_model_info_helper( # noqa: PLR0915 _model_info: Optional[Dict[str, Any]] = None key: Optional[str] = None - if combined_model_name in litellm.model_cost: - key = combined_model_name - _model_info = _get_model_info_from_model_cost(key=cast(str, key)) - if not _check_provider_match( - model_info=_model_info, custom_llm_provider=custom_llm_provider - ): - _model_info = None - if _model_info is None and model in litellm.model_cost: - key = model - _model_info = _get_model_info_from_model_cost(key=cast(str, key)) - if not _check_provider_match( - model_info=_model_info, custom_llm_provider=custom_llm_provider - ): - _model_info = None - if ( - _model_info is None - and combined_stripped_model_name in litellm.model_cost - ): - key = combined_stripped_model_name - _model_info = _get_model_info_from_model_cost(key=cast(str, key)) - if not _check_provider_match( - model_info=_model_info, custom_llm_provider=custom_llm_provider - ): - _model_info = None - if _model_info is None and stripped_model_name in litellm.model_cost: - key = stripped_model_name - _model_info = _get_model_info_from_model_cost(key=cast(str, key)) - if not _check_provider_match( - model_info=_model_info, custom_llm_provider=custom_llm_provider - ): - _model_info = None - if _model_info is None and split_model in litellm.model_cost: - key = split_model + # Use case-insensitive lookup for all model name checks + _matched_key = _get_model_cost_key(combined_model_name) + if _matched_key is not None: + key = _matched_key _model_info = _get_model_info_from_model_cost(key=cast(str, key)) if not _check_provider_match( model_info=_model_info, custom_llm_provider=custom_llm_provider ): _model_info = None + if _model_info is None: + _matched_key = _get_model_cost_key(model) + if _matched_key is not None: + key = _matched_key + _model_info = _get_model_info_from_model_cost(key=cast(str, key)) + if not _check_provider_match( + model_info=_model_info, custom_llm_provider=custom_llm_provider + ): + _model_info = None + if _model_info is None: + _matched_key = _get_model_cost_key(combined_stripped_model_name) + if _matched_key is not None: + key = _matched_key + _model_info = _get_model_info_from_model_cost(key=cast(str, key)) + if not _check_provider_match( + model_info=_model_info, custom_llm_provider=custom_llm_provider + ): + _model_info = None + if _model_info is None: + _matched_key = _get_model_cost_key(stripped_model_name) + if _matched_key is not None: + key = _matched_key + _model_info = _get_model_info_from_model_cost(key=cast(str, key)) + if not _check_provider_match( + model_info=_model_info, custom_llm_provider=custom_llm_provider + ): + _model_info = None + if _model_info is None: + _matched_key = _get_model_cost_key(split_model) + if _matched_key is not None: + key = _matched_key + _model_info = _get_model_info_from_model_cost(key=cast(str, key)) + if not _check_provider_match( + model_info=_model_info, custom_llm_provider=custom_llm_provider + ): + _model_info = None if _model_info is None or key is None: raise ValueError( diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 823f03185d5..b84fc22af01 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -372,3 +372,86 @@ def test_get_model_info_cost_calculator_bedrock_region_cris_stripped(model, prov print("info", info) assert info["key"] == "us.anthropic.claude-3-haiku-20240307-v1:0" assert info["litellm_provider"] == "bedrock" + + +def test_get_model_info_case_insensitive_lookup(monkeypatch): + """ + Test that model info lookup is case-insensitive. + + This ensures that users can use lowercase model names even when the model cost + map has mixed-case keys (e.g., "Qwen/Qwen3-Next-80B-A3B-Thinking"). + + Related Slack discussion: Users were getting "does not support parameters: ['tools']" + errors when using lowercase model names like "qwen/qwen3-next-80b-a3b-thinking" + because the lookup was case-sensitive. + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + + # Register a test model with mixed-case name + litellm.register_model( + { + "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0002, + "litellm_provider": "together_ai", + "supports_function_calling": True, + } + } + ) + + # Test 1: Exact case should work + info = litellm.get_model_info( + model="Qwen/Qwen3-Next-80B-A3B-Thinking", custom_llm_provider="together_ai" + ) + assert info is not None + assert info["supports_function_calling"] is True + + # Test 2: Lowercase should also work (case-insensitive lookup) + info_lower = litellm.get_model_info( + model="qwen/qwen3-next-80b-a3b-thinking", custom_llm_provider="together_ai" + ) + assert info_lower is not None + assert info_lower["supports_function_calling"] is True + + # Test 3: Mixed case should also work + info_mixed = litellm.get_model_info( + model="QWEN/qwen3-NEXT-80b-a3b-thinking", custom_llm_provider="together_ai" + ) + assert info_mixed is not None + assert info_mixed["supports_function_calling"] is True + + +def test_get_model_info_case_insensitive_supports_function_calling(monkeypatch): + """ + Test that supports_function_calling check works with case-insensitive model lookup. + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + + # Register a model with mixed-case name that supports function calling + litellm.register_model( + { + "test_provider/TestModel-ABC": { + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0002, + "litellm_provider": "test_provider", + "supports_function_calling": True, + } + } + ) + + # Test that supports_function_calling works with lowercase model name + from litellm.utils import supports_function_calling + + # Exact case + assert ( + supports_function_calling("TestModel-ABC", custom_llm_provider="test_provider") + is True + ) + + # Lowercase (should now work with case-insensitive lookup) + assert ( + supports_function_calling("testmodel-abc", custom_llm_provider="test_provider") + is True + ) From bf76e66d2ccaace63eb6353ab92e37b57d99b349 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 09:42:59 -0800 Subject: [PATCH 010/699] Working SSO Mapping new user, overrides default user settings --- litellm/proxy/management_endpoints/ui_sso.py | 48 +++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 7a404bcda81..5afccc6fe5f 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -1173,8 +1173,44 @@ async def insert_sso_user( if user_defined_values is None: raise ValueError("user_defined_values is None") + # Check if role_mappings is configured in SSO settings + role_mappings_configured = False + try: + from litellm.proxy.utils import get_prisma_client_or_throw + + prisma_client = get_prisma_client_or_throw( + "Prisma client is None, connect a database to your proxy" + ) + + # Get SSO config from dedicated table + sso_db_record = await prisma_client.db.litellm_ssoconfig.find_unique( + where={"id": "sso_config"} + ) + + if sso_db_record and sso_db_record.sso_settings: + sso_settings_dict = dict(sso_db_record.sso_settings) + role_mappings_data = sso_settings_dict.get("role_mappings") + role_mappings_configured = role_mappings_data is not None + except Exception as e: + # If we can't check role_mappings, continue with existing logic + verbose_proxy_logger.debug( + f"Could not check role_mappings configuration: {e}. Using default behavior." + ) + + # Apply default_internal_user_params if litellm.default_internal_user_params: - user_defined_values.update(litellm.default_internal_user_params) # type: ignore + # If role_mappings is configured and user_role is already set from SSO, preserve it + if role_mappings_configured and user_defined_values.get("user_role") is not None: + # Preserve the SSO-extracted role, but apply other defaults + preserved_role = user_defined_values.get("user_role") + user_defined_values.update(litellm.default_internal_user_params) # type: ignore + user_defined_values["user_role"] = preserved_role # Restore preserved role + verbose_proxy_logger.debug( + f"Preserved SSO-extracted role '{preserved_role}' (role_mappings configured)" + ) + else: + # Default behavior: update all values including role + user_defined_values.update(litellm.default_internal_user_params) # type: ignore # Set budget for internal users if user_defined_values.get("user_role") == LitellmUserRoles.INTERNAL_USER.value: @@ -1812,7 +1848,15 @@ class SSOAuthenticationHandler: ) user_id = getattr(result, "id", None) user_email = getattr(result, "email", None) - user_role = getattr(result, generic_user_role_attribute_name, None) # type: ignore + if user_role is None: + _role_from_attr = getattr(result, generic_user_role_attribute_name, None) # type: ignore + if _role_from_attr is not None: + # Convert enum to string if needed + user_role = ( + _role_from_attr.value + if isinstance(_role_from_attr, LitellmUserRoles) + else _role_from_attr + ) if user_id is None and result is not None: _first_name = getattr(result, "first_name", "") or "" From 313a613a13e2756308ce828924864815ac36bd38 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 10:56:11 -0800 Subject: [PATCH 011/699] Adding tests --- .../proxy/management_endpoints/test_ui_sso.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 500fc67de89..20829466570 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -3043,3 +3043,108 @@ class TestAddMissingTeamMember: assert set(added_teams) == set( expected_teams_added ), f"Expected teams {expected_teams_added}, but got {added_teams}" + + +@pytest.mark.asyncio +async def test_role_mappings_override_default_internal_user_params(): + """ + Test that when role_mappings is configured in SSO settings, + the SSO-extracted role overrides default_internal_user_params role. + """ + from litellm.proxy._types import NewUserResponse, SSOUserDefinedValues + from litellm.proxy.management_endpoints.ui_sso import insert_sso_user + + # Save original default_internal_user_params + original_default_params = getattr(litellm, "default_internal_user_params", None) + + try: + # Set default_internal_user_params with a role that should be overridden + litellm.default_internal_user_params = { + "user_role": "internal_user", + "max_budget": 100, + "budget_duration": "30d", + "models": ["gpt-3.5-turbo"], + } + + # Mock SSO result + mock_result_openid = CustomOpenID( + id="test-user-123", + email="test@example.com", + display_name="Test User", + provider="microsoft", + team_ids=[], + ) + + # User defined values with SSO-extracted role (from role_mappings) + user_defined_values: SSOUserDefinedValues = { + "user_id": "test-user-123", + "user_email": "test@example.com", + "user_role": "proxy_admin", # Role from SSO role_mappings + "max_budget": None, + "budget_duration": None, + "models": [], + } + + # Mock Prisma client with SSO config that has role_mappings configured + mock_prisma = MagicMock() + mock_sso_config = MagicMock() + mock_sso_config.sso_settings = { + "role_mappings": { + "Admin": "proxy_admin", + "User": "internal_user", + } + } + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock( + return_value=mock_sso_config + ) + + # Mock new_user function + mock_new_user_response = NewUserResponse( + user_id="test-user-123", + key="sk-xxxxx", + teams=None, + ) + + with patch( + "litellm.proxy.utils.get_prisma_client_or_throw", + return_value=mock_prisma, + ), patch( + "litellm.proxy.management_endpoints.ui_sso.new_user", + return_value=mock_new_user_response, + ) as mock_new_user: + # Act + result = await insert_sso_user( + result_openid=mock_result_openid, + user_defined_values=user_defined_values, + ) + + # Assert - verify new_user was called with preserved SSO role + mock_new_user.assert_called_once() + call_args = mock_new_user.call_args + new_user_request = call_args.kwargs["data"] + + # The role from SSO should be preserved, not overridden by default_internal_user_params + assert ( + new_user_request.user_role == "proxy_admin" + ), "SSO-extracted role should override default_internal_user_params role" + + # Other default params should still be applied + assert ( + new_user_request.max_budget == 100 + ), "max_budget from default_internal_user_params should be applied" + assert ( + new_user_request.budget_duration == "30d" + ), "budget_duration from default_internal_user_params should be applied" + + # Note: models are applied via _update_internal_new_user_params inside new_user, + # not in insert_sso_user, so we verify user_defined_values was updated correctly + # by checking that the function completed successfully and other defaults were applied + # The models will be applied when new_user processes the request + + finally: + # Restore original default_internal_user_params + if original_default_params is not None: + litellm.default_internal_user_params = original_default_params + else: + if hasattr(litellm, "default_internal_user_params"): + delattr(litellm, "default_internal_user_params") From e92db2d718a78dc0443c801cb8467e6a5160c725 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 13:04:42 -0800 Subject: [PATCH 012/699] Improve e2e flow for cloudzero --- .../hooks/cloudzero/useCloudZeroSettings.ts | 70 +++++++++++++++---- .../CloudZeroCostTracking.tsx | 4 +- .../CloudZeroEmptyPlaceholder.test.tsx | 2 +- .../CloudZeroEmptyPlaceholder.tsx | 2 +- .../CloudZeroIntegrationSettings.tsx | 8 ++- .../components/CloudZeroCostTracking/types.ts | 8 +-- 6 files changed, 71 insertions(+), 23 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts index 5ccbe244e60..96f5ab2f944 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts @@ -17,19 +17,35 @@ const getCloudZeroSettings = async (accessToken: string): Promise ({})); - const errorMessage = - errorData?.error?.message || errorData?.message || errorData?.detail || "Failed to fetch CloudZero settings"; + let errorMessage = "Failed to fetch CloudZero settings"; + try { + const errorData = await response.json(); + // Handle different error response formats + if (typeof errorData === "object" && errorData !== null) { + errorMessage = + errorData?.error?.message || + errorData?.error || + errorData?.message || + errorData?.detail || + (typeof errorData?.error === "string" ? errorData.error : errorMessage); + } else if (typeof errorData === "string") { + errorMessage = errorData; + } + } catch { + // If JSON parsing fails, use the status text + errorMessage = response.statusText || errorMessage; + } throw new Error(errorMessage); } const data = await response.json(); + + // Check if settings are actually configured (all required fields are present) + if (!data || (!data.api_key_masked && !data.connection_id)) { + return null; + } + return data; }; @@ -77,9 +93,22 @@ const updateCloudZeroSettings = async (accessToken: string, params: UpdateParams }); if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - const errorMessage = - errorData?.error?.message || errorData?.message || errorData?.detail || "Failed to update CloudZero settings"; + let errorMessage = "Failed to update CloudZero settings"; + try { + const errorData = await response.json(); + if (typeof errorData === "object" && errorData !== null) { + errorMessage = + errorData?.error?.message || + errorData?.error || + errorData?.message || + errorData?.detail || + (typeof errorData?.error === "string" ? errorData.error : errorMessage); + } else if (typeof errorData === "string") { + errorMessage = errorData; + } + } catch { + errorMessage = response.statusText || errorMessage; + } throw new Error(errorMessage); } @@ -117,9 +146,22 @@ const deleteCloudZeroSettings = async (accessToken: string): Promise ({})); - const errorMessage = - errorData?.error?.message || errorData?.message || errorData?.detail || "Failed to delete CloudZero settings"; + let errorMessage = "Failed to delete CloudZero settings"; + try { + const errorData = await response.json(); + if (typeof errorData === "object" && errorData !== null) { + errorMessage = + errorData?.error?.message || + errorData?.error || + errorData?.message || + errorData?.detail || + (typeof errorData?.error === "string" ? errorData.error : errorMessage); + } else if (typeof errorData === "string") { + errorMessage = errorData; + } + } catch { + errorMessage = response.statusText || errorMessage; + } throw new Error(errorMessage); } diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx index fbb892cb1d8..db3ea94bbf9 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx @@ -36,7 +36,9 @@ export default function CloudZeroCostTracking() { if (error) { return ( - Error loading CloudZero settings: {error.message} + + Error loading CloudZero settings: {error instanceof Error ? error.message : String(error)} + ); } diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx index 04e0a67dea6..f7b90884006 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx @@ -9,6 +9,6 @@ describe("CloudZeroEmptyPlaceholder", () => { expect(screen.getByText("No CloudZero Integration Found")).toBeInTheDocument(); expect(screen.getByText(/Connect your CloudZero account/)).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Create Integration" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Add CloudZero Integration" })).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx index 1719a949b86..aca074dc290 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx @@ -21,7 +21,7 @@ export default function CloudZeroEmptyPlaceholder({ startCreation }: CloudZeroEm } > diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx index 780fa83652a..c161d241f7d 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx @@ -134,10 +134,14 @@ export function CloudZeroIntegrationSettings({ settings, onSettingsUpdated }: Cl }} > - {settings.api_key_masked} + + {settings.api_key_masked || Not configured} + - {settings.connection_id} + + {settings.connection_id || Not configured} + {settings.timezone || Default (UTC)} diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts index a41afee4f72..ed3c76cc3b1 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts @@ -1,6 +1,6 @@ export interface CloudZeroSettings { - api_key_masked: string; - connection_id: string; - timezone?: string; - status?: string; + api_key_masked: string | null; + connection_id: string | null; + timezone?: string | null; + status?: string | null; } From 74842de78eddfdea22448b85aec9a0974b28a36f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 13:04:59 -0800 Subject: [PATCH 013/699] Adding backend --- .../spend_tracking/cloudzero_endpoints.py | 34 ++++-- litellm/types/proxy/cloudzero_endpoints.py | 8 +- .../test_cloudzero_endpoints.py | 111 +++++++++++++++++- 3 files changed, 137 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/spend_tracking/cloudzero_endpoints.py b/litellm/proxy/spend_tracking/cloudzero_endpoints.py index 2cf4ce8f16a..172169f2c7a 100644 --- a/litellm/proxy/spend_tracking/cloudzero_endpoints.py +++ b/litellm/proxy/spend_tracking/cloudzero_endpoints.py @@ -69,7 +69,7 @@ async def _get_cloudzero_settings(): Retrieve CloudZero settings from the database with decrypted API key. Returns: - dict: CloudZero settings with decrypted API key + dict: CloudZero settings with decrypted API key, or empty dict if not configured """ from litellm.proxy.proxy_server import prisma_client @@ -82,10 +82,16 @@ async def _get_cloudzero_settings(): cloudzero_config = await prisma_client.db.litellm_config.find_first( where={"param_name": "cloudzero_settings"} ) - if cloudzero_config is None: + if cloudzero_config is None or cloudzero_config.param_value is None: return {} - settings = dict(cloudzero_config.param_value) + # Handle both dict and JSON string cases + if isinstance(cloudzero_config.param_value, dict): + settings = cloudzero_config.param_value + elif isinstance(cloudzero_config.param_value, str): + settings = json.loads(cloudzero_config.param_value) + else: + settings = dict(cloudzero_config.param_value) # Decrypt the API key encrypted_api_key = settings.get("api_key") @@ -119,6 +125,7 @@ async def get_cloudzero_settings( Returns the current CloudZero configuration with the API key masked for security. Only the first 4 and last 4 characters of the API key are shown. + Returns null/empty values when settings are not configured (consistent with other settings endpoints). Only admin users can view CloudZero settings. """ @@ -133,22 +140,27 @@ async def get_cloudzero_settings( # Get CloudZero settings using the accessor method settings = await _get_cloudzero_settings() + # If settings are empty, return null/empty values (consistent with other endpoints) + if not settings: + return CloudZeroSettingsView( + api_key_masked=None, + connection_id=None, + timezone=None, + status=None, + ) + # Use SensitiveDataMasker to mask the API key masked_settings = _sensitive_masker.mask_dict(settings) return CloudZeroSettingsView( - api_key_masked=masked_settings["api_key"], - connection_id=settings["connection_id"], - timezone=settings["timezone"], + api_key_masked=masked_settings.get("api_key"), + connection_id=settings.get("connection_id"), + timezone=settings.get("timezone"), status="configured", ) except HTTPException as e: - if e.status_code == 400: - # Settings not configured - raise HTTPException( - status_code=404, detail={"error": "CloudZero settings not configured"} - ) + # Re-raise HTTPExceptions as-is raise e except Exception as e: verbose_proxy_logger.error(f"Error retrieving CloudZero settings: {str(e)}") diff --git a/litellm/types/proxy/cloudzero_endpoints.py b/litellm/types/proxy/cloudzero_endpoints.py index 1d909bf7f8c..fc48717e80a 100644 --- a/litellm/types/proxy/cloudzero_endpoints.py +++ b/litellm/types/proxy/cloudzero_endpoints.py @@ -45,10 +45,10 @@ class CloudZeroExportResponse(BaseModel): class CloudZeroSettingsView(BaseModel): """Response model for viewing CloudZero settings with masked API key""" - api_key_masked: str = Field(..., description="Masked API key showing only first 4 and last 4 characters") - connection_id: str = Field(..., description="CloudZero connection ID for data submission") - timezone: str = Field(..., description="Timezone for date handling") - status: str = Field(..., description="Configuration status") + api_key_masked: Optional[str] = Field(None, description="Masked API key showing only first 4 and last 4 characters") + connection_id: Optional[str] = Field(None, description="CloudZero connection ID for data submission") + timezone: Optional[str] = Field(None, description="Timezone for date handling") + status: Optional[str] = Field(None, description="Configuration status") class CloudZeroSettingsUpdate(BaseModel): diff --git a/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py index 8ff5774bf50..6d460f63332 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py @@ -1,6 +1,6 @@ import os import sys -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient @@ -77,3 +77,112 @@ async def test_delete_cloudzero_settings_not_found(client, monkeypatch): finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + +@pytest.mark.asyncio +async def test_get_cloudzero_settings_success(client, monkeypatch): + """Test GET /cloudzero/settings returns settings when configured""" + mock_config = MagicMock() + mock_config.param_name = "cloudzero_settings" + mock_config.param_value = { + "api_key": "encrypted_key", + "connection_id": "conn_123", + "timezone": "UTC" + } + + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=mock_config) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + # Mock the decrypt function to return a decrypted key + with patch("litellm.proxy.spend_tracking.cloudzero_endpoints.decrypt_value_helper") as mock_decrypt: + mock_decrypt.return_value = "decrypted_api_key" + + # Mock the masker + with patch("litellm.proxy.spend_tracking.cloudzero_endpoints._sensitive_masker") as mock_masker: + mock_masker.mask_dict.return_value = {"api_key": "test****key"} + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.get("/cloudzero/settings") + assert response.status_code == 200 + data = response.json() + assert data["connection_id"] == "conn_123" + assert data["timezone"] == "UTC" + assert data["status"] == "configured" + assert data["api_key_masked"] == "test****key" + mock_litellm_config.find_first.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_get_cloudzero_settings_not_configured(client, monkeypatch): + """Test GET /cloudzero/settings returns 200 with null values when not configured (consistent with other endpoints)""" + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=None) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.get("/cloudzero/settings") + # Should return 200 with null values (not 404) - consistent with other settings endpoints + assert response.status_code == 200 + data = response.json() + assert data["api_key_masked"] is None + assert data["connection_id"] is None + assert data["timezone"] is None + assert data["status"] is None + mock_litellm_config.find_first.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_get_cloudzero_settings_empty_param_value(client, monkeypatch): + """Test GET /cloudzero/settings returns 200 with null values when param_value is None""" + mock_config = MagicMock() + mock_config.param_name = "cloudzero_settings" + mock_config.param_value = None + + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=mock_config) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.get("/cloudzero/settings") + # Should return 200 with null values (not 404) - consistent with other settings endpoints + assert response.status_code == 200 + data = response.json() + assert data["api_key_masked"] is None + assert data["connection_id"] is None + assert data["timezone"] is None + assert data["status"] is None + mock_litellm_config.find_first.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + From ffcac2eebcc3a1f2c74a81171d79c930750f0ee8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 18:04:04 -0800 Subject: [PATCH 014/699] Allow deleting key expiry --- .../key_management_endpoints.py | 13 ++++++-- tests/proxy_unit_tests/test_proxy_utils.py | 5 +++ .../test_key_management_endpoints.py | 31 +++++++++++++++++++ .../KeyLifecycleSettings.tsx | 4 +-- 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 14d221d19e1..cc2ac908149 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -507,7 +507,11 @@ async def _common_key_generation_helper( # noqa: PLR0915 upperbound_duration = duration_in_seconds( duration=upperbound_value ) - user_duration = duration_in_seconds(duration=value) + # Handle special case where duration is "-1" (never expires) + if value == "-1": + user_duration = float('inf') # Infinite duration + else: + user_duration = duration_in_seconds(duration=value) if user_duration > upperbound_duration: raise HTTPException( status_code=400, @@ -1339,7 +1343,10 @@ async def prepare_key_update_data( if "duration" in non_default_values: duration = non_default_values.pop("duration") - if duration and (isinstance(duration, str)) and len(duration) > 0: + if duration == "-1": + # Set expires to None to indicate the key never expires + non_default_values["expires"] = None + elif duration and (isinstance(duration, str)) and len(duration) > 0: duration_s = duration_in_seconds(duration=duration) expires = datetime.now(timezone.utc) + timedelta(seconds=duration_s) non_default_values["expires"] = expires @@ -1452,7 +1459,7 @@ async def update_key_fn( - tpm_limit_type: Optional[str] - TPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic" - rpm_limit_type: Optional[str] - RPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic" - allowed_cache_controls: Optional[list] - List of allowed cache control values - - duration: Optional[str] - Key validity duration ("30d", "1h", etc.) + - duration: Optional[str] - Key validity duration ("30d", "1h", etc.) or "-1" to never expire - permissions: Optional[dict] - Key-specific permissions - send_invite_email: Optional[bool] - Send invite email to user_id - guardrails: Optional[List[str]] - List of active guardrails for the key diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 2e5cfff8bf0..5c3c3948920 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -678,6 +678,11 @@ async def test_prepare_key_update_data(): updated_data = await prepare_key_update_data(data, existing_key_row) assert updated_data["metadata"] is None + # Test duration "-1" sets expires to None (never expires) + data = UpdateKeyRequest(key="test_key", duration="-1") + updated_data = await prepare_key_update_data(data, existing_key_row) + assert updated_data["expires"] is None + @pytest.mark.parametrize( "env_vars, expected_url", diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index ff85e6d9e73..648045a7ea6 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -815,6 +815,37 @@ async def test_update_service_account_works_with_team_id(): await prepare_key_update_data(data=data, existing_key_row=existing_key) +@pytest.mark.asyncio +async def test_prepare_key_update_data_duration_never_expires(): + """Test that duration="-1" sets expires to None (never expires).""" + from litellm.proxy._types import UpdateKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import ( + prepare_key_update_data, + ) + + # Mock existing key + existing_key = LiteLLM_VerificationToken( + token="test-token", + key_alias="test-key", + models=["gpt-3.5-turbo"], + user_id="test-user", + team_id=None, + auto_rotate=False, + rotation_interval=None, + metadata={}, + ) + + # Test setting duration to "-1" (never expires) + update_request = UpdateKeyRequest(key="test-token", duration="-1") + + result = await prepare_key_update_data( + data=update_request, existing_key_row=existing_key + ) + + # Verify that expires is set to None + assert result["expires"] is None + + @pytest.mark.asyncio async def test_validate_team_id_used_in_service_account_request_requires_team_id(): """ diff --git a/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.tsx b/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.tsx index 8129f4314fc..81d22b56347 100644 --- a/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.tsx +++ b/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.tsx @@ -64,13 +64,13 @@ const KeyLifecycleSettings: React.FC = ({
Date: Sat, 20 Dec 2025 11:35:20 +0800 Subject: [PATCH 015/699] fix: fixed the issue of handling root paths when processing Discovery protected resource metadata and authorization server metadata URLs. --- .../mcp_server/discoverable_endpoints.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index ffa17a5b7c4..4b6020f582b 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -15,6 +15,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.types.mcp_server.mcp_server_manager import MCPServer +from litellm.proxy.utils import get_server_root_path router = APIRouter( tags=["mcp"], @@ -381,7 +382,18 @@ async def callback(code: str, state: str): # ------------------------------ # Optional .well-known endpoints for MCP + OAuth discovery # ------------------------------ -@router.get("/.well-known/oauth-protected-resource/{mcp_server_name}/mcp") +""" + Per SEP-985, the client MUST: + 1. Try resource_metadata from WWW-Authenticate header (if present) + 2. Fall back to path-based well-known URI: /.well-known/oauth-protected-resource/{path} + ( + If the resource identifier value contains a path or query component, any terminating slash (/) + following the host component MUST be removed before inserting /.well-known/ and the well-known + URI path suffix between the host component and the path(include root path) and/or query components. + https://datatracker.ietf.org/doc/html/rfc9728#section-3.1) + 3. Fall back to root-based well-known URI: /.well-known/oauth-protected-resource +""" +@router.get(f"/.well-known/oauth-protected-resource{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}/mcp") @router.get("/.well-known/oauth-protected-resource") async def oauth_protected_resource_mcp( request: Request, mcp_server_name: Optional[str] = None @@ -403,8 +415,15 @@ async def oauth_protected_resource_mcp( ), # this is what Claude will call } - -@router.get("/.well-known/oauth-authorization-server/{mcp_server_name}") +""" + https://datatracker.ietf.org/doc/html/rfc8414#section-3.1 + RFC 8414: Path-aware OAuth discovery + If the issuer identifier value contains a path component, any + terminating "/" MUST be removed before inserting "/.well-known/" and + the well-known URI suffix between the host component and the path(include root path) + component. +""" +@router.get(f"/.well-known/oauth-authorization-server{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}") @router.get("/.well-known/oauth-authorization-server") async def oauth_authorization_server_mcp( request: Request, mcp_server_name: Optional[str] = None From 3a2ab6b0d12be8863d2a7604a1f3e8ab5721b521 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 11:55:12 +0800 Subject: [PATCH 016/699] fix: added additional grant type into oauth_authorization_server response for fixing mcp auth register bad request issue --- .../proxy/_experimental/mcp_server/discoverable_endpoints.py | 2 +- .../_experimental/mcp_server/test_discoverable_endpoints.py | 3 ++- ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index ffa17a5b7c4..5433196dfe3 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -428,7 +428,7 @@ async def oauth_authorization_server_mcp( "authorization_endpoint": authorization_endpoint, "token_endpoint": token_endpoint, "response_types_supported": ["code"], - "grant_types_supported": ["authorization_code"], + "grant_types_supported": ["authorization_code", "refresh_token"], "code_challenge_methods_supported": ["S256"], "token_endpoint_auth_methods_supported": ["client_secret_post"], # Claude expects a registration endpoint, even if we just fake it diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 6df9abd3fee..30f3d55f028 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -354,7 +354,7 @@ async def test_register_client_remote_registration_success(): request_payload = { "client_name": "Litellm Proxy", - "grant_types": ["authorization_code"], + "grant_types": ["authorization_code", "refresh_token"], "response_types": ["code"], "token_endpoint_auth_method": "client_secret_post", } @@ -603,6 +603,7 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): assert response["authorization_endpoint"].startswith("https://litellm.example.com/") assert response["token_endpoint"].startswith("https://litellm.example.com/") assert response["registration_endpoint"].startswith("https://litellm.example.com/") + assert response["grant_types_supported"] == ["authorization_code", "refresh_token"] @pytest.mark.asyncio diff --git a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx index 9600c962564..a62d8baa75b 100644 --- a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx +++ b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx @@ -136,7 +136,7 @@ export const useMcpOAuthFlow = ({ if (!hasPreconfiguredCredentials) { const registration = await registerMcpOAuthClient(accessToken, serverId, { client_name: temporaryPayload.alias || temporaryPayload.server_name || serverId, - grant_types: ["authorization_code"], + grant_types: ["authorization_code", "refresh_token"], response_types: ["code"], token_endpoint_auth_method: temporaryPayload.credentials && temporaryPayload.credentials.client_secret ? "client_secret_post" : "none", From 684fba42eaaf6a4d47795e56fd668b8d46b01525 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 13:22:29 +0800 Subject: [PATCH 017/699] fix: added RFC RECOMMENDED property(scopes_supported) to protected resource and authorization server metadata --- .../mcp_server/discoverable_endpoints.py | 17 +++++- .../mcp_server/test_discoverable_endpoints.py | 54 ++++++++++++++++++- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index d6fe3f2b9cf..ded591a8f53 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -398,8 +398,14 @@ async def callback(code: str, state: str): async def oauth_protected_resource_mcp( request: Request, mcp_server_name: Optional[str] = None ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) # Get the correct base URL considering X-Forwarded-* headers request_base_url = get_request_base_url(request) + mcp_server: Optional[MCPServer] = None + if mcp_server_name: + mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name) return { "authorization_servers": [ ( @@ -413,6 +419,7 @@ async def oauth_protected_resource_mcp( if mcp_server_name else f"{request_base_url}/mcp" ), # this is what Claude will call + "scopes_supported": mcp_server.scopes if mcp_server else [], } """ @@ -428,6 +435,9 @@ async def oauth_protected_resource_mcp( async def oauth_authorization_server_mcp( request: Request, mcp_server_name: Optional[str] = None ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) # Get the correct base URL considering X-Forwarded-* headers request_base_url = get_request_base_url(request) @@ -442,16 +452,21 @@ async def oauth_authorization_server_mcp( else f"{request_base_url}/token" ) + mcp_server: Optional[MCPServer] = None + if mcp_server_name: + mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name) + return { "issuer": request_base_url, # point to your proxy "authorization_endpoint": authorization_endpoint, "token_endpoint": token_endpoint, "response_types_supported": ["code"], + "scopes_supported": mcp_server.scopes if mcp_server else [], "grant_types_supported": ["authorization_code", "refresh_token"], "code_challenge_methods_supported": ["S256"], "token_endpoint_auth_methods_supported": ["client_secret_post"], # Claude expects a registration endpoint, even if we just fake it - "registration_endpoint": f"{request_base_url}/{mcp_server_name}/register", + "registration_endpoint": f"{request_base_url}/{mcp_server_name}/register" if mcp_server_name else f"{request_base_url}/register", } diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 30f3d55f028..4c5723b8284 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -556,9 +556,33 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( oauth_protected_resource_mcp, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_client_secret", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) @@ -568,13 +592,14 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): # Call the endpoint response = await oauth_protected_resource_mcp( request=mock_request, - mcp_server_name="test_server", + mcp_server_name="test_oauth", ) # Verify response uses HTTPS URLs assert response["authorization_servers"][0].startswith( "https://litellm.example.com/" ) + assert response["scopes_supported"] == oauth2_server.scopes @pytest.mark.asyncio @@ -584,9 +609,33 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( oauth_authorization_server_mcp, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_client_secret", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) @@ -596,7 +645,7 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): # Call the endpoint response = await oauth_authorization_server_mcp( request=mock_request, - mcp_server_name="test_server", + mcp_server_name="test_oauth", ) # Verify response uses HTTPS URLs @@ -604,6 +653,7 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): assert response["token_endpoint"].startswith("https://litellm.example.com/") assert response["registration_endpoint"].startswith("https://litellm.example.com/") assert response["grant_types_supported"] == ["authorization_code", "refresh_token"] + assert response["scopes_supported"] == oauth2_server.scopes @pytest.mark.asyncio From 0306f02e74d7fff462fb727639a60e5ae11d64e4 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 14:13:18 +0800 Subject: [PATCH 018/699] fix: removed initialize the tool name to MCP server name mapping(oauth2) on startup for avoiding 401 error --- .../mcp_server/mcp_server_manager.py | 3 +++ .../mcp_server/test_mcp_server_manager.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 8c9d8630457..c2215efe9d0 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1913,6 +1913,9 @@ class MCPServerManager: Note: This now handles prefixed tool names """ for server in self.get_registry().values(): + if server.auth_type == MCPAuth.oauth2: + # Skip OAuth2 servers for now as they may require user-specific tokens + continue tools = await self._get_tools_from_server(server) for tool in tools: # The tool.name here is already prefixed from _get_tools_from_server diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 7a6e5ad17f6..c0ded9c728c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -536,7 +536,26 @@ class TestMCPServerManager: assert ( server.registration_url == "https://discovered.example.com/register" ) + @pytest.mark.asyncio + async def test_config_oauth_initialize_tool_name_to_mcp_server_name_mapping(self): + manager = MCPServerManager() + config = { + "example": { + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + "auth_type": MCPAuth.oauth2, + "scopes": ["config"], + "authorization_url": "https://config.example.com/auth", + } + } + + await manager.load_servers_from_config(config) + + # Initialize the tool mapping + await manager._initialize_tool_name_to_mcp_server_name_mapping() + assert manager.tool_name_to_mcp_server_name_mapping == {} + @pytest.mark.asyncio async def test_list_tools_handles_missing_server_alias(self): """Test that list_tools handles servers without alias gracefully""" From 36a369a747fccedb87e12cf26996aebfc221f57e Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 14:28:52 +0800 Subject: [PATCH 019/699] fix: upgraded mcp sdk depency version for fixing ClosedResourceError --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f222acc46e6..cb12a658814 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ google-cloud-aiplatform==1.47.0 # for vertex ai calls google-cloud-iam==2.19.1 # for GCP IAM Redis authentication google-genai==1.22.0 anthropic[vertex]==0.54.0 -mcp==1.21.2 ; python_version >= "3.10" # for MCP server +mcp==1.25.0 ; python_version >= "3.10" # for MCP server google-generativeai==0.5.0 # for vertex ai calls async_generator==1.10.0 # for async ollama calls langfuse==2.59.7 # for langfuse self-hosted logging From b981fddfaadd95d97dca9df4be3561d5a6f9a2cf Mon Sep 17 00:00:00 2001 From: Lucas Rothman Date: Fri, 19 Dec 2025 23:13:36 -0800 Subject: [PATCH 020/699] fix(gemini): properly catch context window exceeded errors Fixes #18282 This PR fixes two issues with Gemini context window error handling: 1. **Pattern matching for Gemini 2.0 Flash**: The previous pattern 'input token count exceeds the maximum number of tokens allowed' doesn't match Gemini 2.0 Flash errors which include dynamic token counts like '(2800010)' in the message. Split into shorter patterns that work with both formats. 2. **Add context window check to Gemini block**: The is_error_str_context_window_exceeded() check was only called for OpenAI-compatible providers, not for Gemini/Vertex AI. Added the check to the Gemini-specific error handling block. Test cases added for both Gemini 2.0 Flash and 2.5/3 error formats. --- .../exception_mapping_utils.py | 12 +++- .../test_exception_mapping_utils.py | 60 ++++++++++++++++++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 7bf95ca3404..1517d1e776d 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -78,9 +78,7 @@ class ExceptionCheckers: "is longer than the model's context length", "input tokens exceed the configured limit", "`inputs` tokens + `max_new_tokens` must be", - # Gemini pattern: "The input token count exceeds the maximum number of tokens allowed" - # See: https://github.com/BerriAI/litellm/issues/XXXX - "input token count exceeds the maximum number of tokens allowed", + "exceeds the maximum number of tokens allowed", # Gemini ] for substring in known_exception_substrings: if substring in _error_str_lowercase: @@ -1262,6 +1260,14 @@ def exception_type( # type: ignore # noqa: PLR0915 model=model, llm_provider=custom_llm_provider, ) + elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"ContextWindowExceededError: {custom_llm_provider.capitalize()}Exception - {error_str}", + model=model, + llm_provider=custom_llm_provider, + litellm_debug_info=extra_information, + ) elif ( "None Unknown Error." in error_str or "Content has no parts." in error_str diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index f69b9c35236..9e742a83c6a 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -40,8 +40,7 @@ context_window_test_cases = [ "`inputs` tokens + `max_new_tokens` must be <= 4096", True, ), - # Gemini context window error format - # See: https://github.com/BerriAI/litellm/issues/XXXX + # Gemini 2.5/3 format ( "The input token count exceeds the maximum number of tokens allowed 1048576.", True, @@ -50,6 +49,15 @@ context_window_test_cases = [ "GeminiException BadRequestError - {\n \"error\": {\n \"code\": 400,\n \"message\": \"The input token count exceeds the maximum number of tokens allowed 1048576.\",\n \"status\": \"INVALID_ARGUMENT\"\n }\n}\n", True, ), + # Gemini 2.0 Flash format (includes input token count in message) + ( + "The input token count (2800010) exceeds the maximum number of tokens allowed (1048575).", + True, + ), + ( + "GeminiException BadRequestError - {\n \"error\": {\n \"code\": 400,\n \"message\": \"The input token count (2800010) exceeds the maximum number of tokens allowed (1048575).\",\n \"status\": \"INVALID_ARGUMENT\"\n }\n}\n", + True, + ), # Test case insensitivity ("ERROR: THIS MODEL'S MAXIMUM CONTEXT LENGTH IS 1024.", True), # Cerebras context window error format @@ -169,6 +177,54 @@ class TestExceptionCheckers: result = ExceptionCheckers.is_azure_content_policy_violation_error(error_str) assert result is False, f"Should NOT detect policy violation in: {error_str}" +gemini_context_window_test_cases = [ + # Gemini 2.0 Flash format (includes input token count in message) + ( + "The input token count (2800010) exceeds the maximum number of tokens allowed (1048575).", + True, + ), + # Gemini 2.5/3 format + ( + "The input token count exceeds the maximum number of tokens allowed (1048576).", + True, + ), + ("A generic error occurred.", False), +] + + +@pytest.mark.parametrize( + "error_message, should_raise_context_window", gemini_context_window_test_cases +) +def test_gemini_context_window_error_mapping(error_message, should_raise_context_window): + """ + Tests that the exception_type function correctly maps Gemini's + context window exceeded errors to litellm.ContextWindowExceededError. + """ + model = "gemini/gemini-2.0-flash" + custom_llm_provider = "gemini" + + # Create a generic exception with the specific error message + original_exception = Exception(error_message) + + if should_raise_context_window: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model=model, + original_exception=original_exception, + custom_llm_provider=custom_llm_provider, + ) + # Check if the raised exception is indeed a ContextWindowExceededError + assert isinstance(excinfo.value, litellm.ContextWindowExceededError) + else: + # For the negative case, we expect it to raise a generic APIConnectionError + with pytest.raises(litellm.APIConnectionError): + exception_type( + model=model, + original_exception=original_exception, + custom_llm_provider=custom_llm_provider, + ) + + # Test cases for Vertex AI RateLimitError mapping # As per https://github.com/BerriAI/litellm/issues/16189 vertex_rate_limit_test_cases = [ From 9002f75277228c0723d0d503f71f1f1a5de4638f Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 20 Dec 2025 11:01:41 -0600 Subject: [PATCH 021/699] Require auth for MCP connection test --- .../mcp_server/rest_endpoints.py | 8 +-- .../mcp_server/test_rest_endpoints.py | 55 +++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 032331ece02..92c390f0a64 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1,5 +1,4 @@ import importlib -import traceback from typing import Dict, List, Optional, Union from fastapi import APIRouter, Depends, Query, Request @@ -329,16 +328,15 @@ if MCP_AVAILABLE: except Exception as e: verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True) - stack_trace = traceback.format_exc() return { "status": "error", - "message": f"An internal error has occurred: {str(e)}", - "stack_trace": stack_trace, + "message": "An internal error has occurred while testing the MCP server.", } - @router.post("/test/connection") + @router.post("/test/connection", dependencies=[Depends(user_api_key_auth)]) async def test_connection( request: NewMCPServerRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ Test if we can connect to the provided MCP server before adding it diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index a0c09663a88..85ec807b1ff 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -7,6 +7,7 @@ from litellm.proxy._experimental.mcp_server import rest_endpoints from litellm.proxy._experimental.mcp_server.auth import ( user_api_key_auth_mcp as auth_mcp, ) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy._types import NewMCPServerRequest, UserAPIKeyAuth from litellm.types.mcp import MCPAuth @@ -31,6 +32,60 @@ def _build_request(headers: Optional[Dict[str, str]] = None) -> Request: return Request(scope, receive=receive) +def _get_route(path: str, method: str): + for route in rest_endpoints.router.routes: + if getattr(route, "path", None) == path and method in getattr( + route, "methods", set() + ): + return route + raise AssertionError(f"Route {method} {path} not found") + + +def _route_has_dependency(route, dependency) -> bool: + if any( + getattr(dep, "dependency", None) == dependency + for dep in getattr(route, "dependencies", []) + ): + return True + dependant = getattr(route, "dependant", None) + if dependant is None: + return False + return any(getattr(dep, "call", None) == dependency for dep in dependant.dependencies) + + +@pytest.mark.asyncio +async def test_execute_with_mcp_client_redacts_stack_trace(monkeypatch): + def fake_create_client(*args, **kwargs): + return object() + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_create_mcp_client", + fake_create_client, + ) + + async def failing_operation(client): + raise RuntimeError("boom") + + payload = NewMCPServerRequest( + server_name="example", + url="https://example.com", + auth_type=MCPAuth.none, + ) + + result = await rest_endpoints._execute_with_mcp_client( + payload, failing_operation + ) + + assert result["status"] == "error" + assert "stack_trace" not in result + + +def test_test_connection_requires_auth_dependency(): + route = _get_route("/mcp-rest/test/connection", "POST") + assert _route_has_dependency(route, user_api_key_auth) + + @pytest.mark.asyncio async def test_test_tools_list_forwards_mcp_auth_header(monkeypatch): """Ensure credential-based auth forwards the auth_value to the MCP client.""" From b54216d05f5866b9cf4fce5f6e1e41b4527e1869 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sun, 21 Dec 2025 05:46:34 +0900 Subject: [PATCH 022/699] fix: ensure Datadog callback runs alongside LLM Observability --- litellm/integrations/datadog/datadog.py | 133 +++++++----------- .../integrations/datadog/datadog_handler.py | 50 +++++++ .../integrations/datadog/datadog_llm_obs.py | 15 +- .../test_litellm_logging.py | 34 +++++ 4 files changed, 139 insertions(+), 93 deletions(-) create mode 100644 litellm/integrations/datadog/datadog_handler.py diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 21e1d562224..08bf5034f0e 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -27,6 +27,12 @@ import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.integrations.datadog.datadog_handler import ( + get_datadog_hostname, + get_datadog_service, + get_datadog_source, + get_datadog_tags, +) from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, @@ -67,23 +73,23 @@ class DataDogLogger( Optional environment variables (DataDog Agent): `LITELLM_DD_AGENT_HOST` - hostname or IP of DataDog agent, example = `"localhost"` `LITELLM_DD_AGENT_PORT` - port of DataDog agent (default: 10518 for logs) - + Note: We use LITELLM_DD_AGENT_HOST instead of DD_AGENT_HOST to avoid conflicts with ddtrace which automatically sets DD_AGENT_HOST for APM tracing. """ try: verbose_logger.debug("Datadog: in init datadog logger") - + ######################################################### # Handle datadog_params set as litellm.datadog_params ######################################################### dict_datadog_params = self._get_datadog_params() kwargs.update(dict_datadog_params) - + self.async_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) - + # Configure DataDog endpoint (Agent or Direct API) # Use LITELLM_DD_AGENT_HOST to avoid conflicts with ddtrace's DD_AGENT_HOST dd_agent_host = os.getenv("LITELLM_DD_AGENT_HOST") @@ -91,7 +97,7 @@ class DataDogLogger( self._configure_dd_agent(dd_agent_host=dd_agent_host) else: self._configure_dd_direct_api() - + # Optional override for testing self._apply_dd_base_url_override() self.sync_client = _get_httpx_client() @@ -118,17 +124,21 @@ class DataDogLogger( dict_datadog_params = litellm.datadog_params.model_dump() elif isinstance(litellm.datadog_params, Dict): # only allow params that are of DatadogInitParams - dict_datadog_params = DatadogInitParams(**litellm.datadog_params).model_dump() + dict_datadog_params = DatadogInitParams( + **litellm.datadog_params + ).model_dump() return dict_datadog_params def _configure_dd_agent(self, dd_agent_host: str) -> None: """ Configure DataDog Agent for log forwarding - + Args: dd_agent_host: Hostname or IP of DataDog agent """ - dd_agent_port = os.getenv("LITELLM_DD_AGENT_PORT", "10518") # default port for logs + dd_agent_port = os.getenv( + "LITELLM_DD_AGENT_PORT", "10518" + ) # default port for logs self.intake_url = f"http://{dd_agent_host}:{dd_agent_port}/api/v2/logs" self.DD_API_KEY = os.getenv("DD_API_KEY") # Optional when using agent verbose_logger.debug(f"Datadog: Using DD Agent at {self.intake_url}") @@ -136,7 +146,7 @@ class DataDogLogger( def _configure_dd_direct_api(self) -> None: """ Configure direct DataDog API connection - + Raises: Exception: If required environment variables are not set """ @@ -144,11 +154,9 @@ class DataDogLogger( raise Exception("DD_API_KEY is not set, set 'DD_API_KEY=<>") if os.getenv("DD_SITE", None) is None: raise Exception("DD_SITE is not set in .env, set 'DD_SITE=<>") - + self.DD_API_KEY = os.getenv("DD_API_KEY") - self.intake_url = ( - f"https://http-intake.logs.{os.getenv('DD_SITE')}/api/v2/logs" - ) + self.intake_url = f"https://http-intake.logs.{os.getenv('DD_SITE')}/api/v2/logs" def _apply_dd_base_url_override(self) -> None: """ @@ -270,7 +278,7 @@ class DataDogLogger( # Add API key if available (required for direct API, optional for agent) if self.DD_API_KEY: headers["DD-API-KEY"] = self.DD_API_KEY - + response = self.sync_client.post( url=self.intake_url, json=dd_payload, # type: ignore @@ -318,16 +326,15 @@ class DataDogLogger( status: DataDogStatus, ) -> DatadogPayload: from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + json_payload = safe_dumps(standard_logging_object) verbose_logger.debug("Datadog: Logger - Logging payload = %s", json_payload) dd_payload = DatadogPayload( - ddsource=self._get_datadog_source(), - ddtags=self._get_datadog_tags( - standard_logging_object=standard_logging_object - ), - hostname=self._get_datadog_hostname(), + ddsource=get_datadog_source(), + ddtags=get_datadog_tags(standard_logging_object=standard_logging_object), + hostname=get_datadog_hostname(), message=json_payload, - service=self._get_datadog_service(), + service=get_datadog_service(), status=status, ) return dd_payload @@ -384,18 +391,19 @@ class DataDogLogger( import gzip from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + compressed_data = gzip.compress(safe_dumps(data).encode("utf-8")) - + # Build headers headers = { "Content-Encoding": "gzip", "Content-Type": "application/json", } - + # Add API key if available (required for direct API, optional for agent) if self.DD_API_KEY: headers["DD-API-KEY"] = self.DD_API_KEY - + response = await self.async_client.post( url=self.intake_url, data=compressed_data, # type: ignore @@ -421,13 +429,14 @@ class DataDogLogger( _payload_dict = payload.model_dump() _payload_dict.update(event_metadata or {}) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + _dd_message_str = safe_dumps(_payload_dict) _dd_payload = DatadogPayload( - ddsource=self._get_datadog_source(), - ddtags=self._get_datadog_tags(), - hostname=self._get_datadog_hostname(), + ddsource=get_datadog_source(), + ddtags=get_datadog_tags(), + hostname=get_datadog_hostname(), message=_dd_message_str, - service=self._get_datadog_service(), + service=get_datadog_service(), status=DataDogStatus.WARN, ) @@ -462,13 +471,14 @@ class DataDogLogger( _payload_dict.update(event_metadata or {}) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + _dd_message_str = safe_dumps(_payload_dict) _dd_payload = DatadogPayload( - ddsource=self._get_datadog_source(), - ddtags=self._get_datadog_tags(), - hostname=self._get_datadog_hostname(), + ddsource=get_datadog_source(), + ddtags=get_datadog_tags(), + hostname=get_datadog_hostname(), message=_dd_message_str, - service=self._get_datadog_service(), + service=get_datadog_service(), status=DataDogStatus.INFO, ) @@ -530,7 +540,6 @@ class DataDogLogger( else: clean_metadata[key] = value - # Build the initial payload payload = { "id": id, @@ -550,69 +559,21 @@ class DataDogLogger( } from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + json_payload = safe_dumps(payload) verbose_logger.debug("Datadog: Logger - Logging payload = %s", json_payload) dd_payload = DatadogPayload( - ddsource=self._get_datadog_source(), - ddtags=self._get_datadog_tags(), - hostname=self._get_datadog_hostname(), + ddsource=get_datadog_source(), + ddtags=get_datadog_tags(), + hostname=get_datadog_hostname(), message=json_payload, - service=self._get_datadog_service(), + service=get_datadog_service(), status=DataDogStatus.INFO, ) return dd_payload - @staticmethod - def _get_datadog_tags( - standard_logging_object: Optional[StandardLoggingPayload] = None, - ) -> str: - """ - Get the datadog tags for the request - - DD tags need to be as follows: - - tags: ["user_handle:dog@gmail.com", "app_version:1.0.0"] - """ - base_tags = { - "env": os.getenv("DD_ENV", "unknown"), - "service": os.getenv("DD_SERVICE", "litellm"), - "version": os.getenv("DD_VERSION", "unknown"), - "HOSTNAME": DataDogLogger._get_datadog_hostname(), - "POD_NAME": os.getenv("POD_NAME", "unknown"), - } - - tags = [f"{k}:{v}" for k, v in base_tags.items()] - - if standard_logging_object: - _request_tags: List[str] = ( - standard_logging_object.get("request_tags", []) or [] - ) - request_tags = [f"request_tag:{tag}" for tag in _request_tags] - tags.extend(request_tags) - - return ",".join(tags) - - @staticmethod - def _get_datadog_source(): - return os.getenv("DD_SOURCE", "litellm") - - @staticmethod - def _get_datadog_service(): - return os.getenv("DD_SERVICE", "litellm-server") - - @staticmethod - def _get_datadog_hostname(): - return os.getenv("HOSTNAME", "") - - @staticmethod - def _get_datadog_env(): - return os.getenv("DD_ENV", "unknown") - - @staticmethod - def _get_datadog_pod_name(): - return os.getenv("POD_NAME", "unknown") - async def async_health_check(self) -> IntegrationHealthCheckStatus: """ Check if the service is healthy @@ -651,4 +612,4 @@ class DataDogLogger( start_time_utc: Optional[datetimeObj], end_time_utc: Optional[datetimeObj], ) -> Optional[dict]: - pass \ No newline at end of file + pass diff --git a/litellm/integrations/datadog/datadog_handler.py b/litellm/integrations/datadog/datadog_handler.py new file mode 100644 index 00000000000..26fab77759e --- /dev/null +++ b/litellm/integrations/datadog/datadog_handler.py @@ -0,0 +1,50 @@ +"""Shared helpers for Datadog integrations.""" + +from __future__ import annotations + +import os +from typing import List, Optional + +from litellm.types.utils import StandardLoggingPayload + + +def get_datadog_source() -> str: + return os.getenv("DD_SOURCE", "litellm") + + +def get_datadog_service() -> str: + return os.getenv("DD_SERVICE", "litellm-server") + + +def get_datadog_hostname() -> str: + return os.getenv("HOSTNAME", "") + + +def get_datadog_env() -> str: + return os.getenv("DD_ENV", "unknown") + + +def get_datadog_pod_name() -> str: + return os.getenv("POD_NAME", "unknown") + + +def get_datadog_tags( + standard_logging_object: Optional[StandardLoggingPayload] = None, +) -> str: + """Build Datadog tags string used by multiple integrations.""" + + base_tags = { + "env": get_datadog_env(), + "service": get_datadog_service(), + "version": os.getenv("DD_VERSION", "unknown"), + "HOSTNAME": get_datadog_hostname(), + "POD_NAME": get_datadog_pod_name(), + } + + tags: List[str] = [f"{k}:{v}" for k, v in base_tags.items()] + + if standard_logging_object: + request_tags = standard_logging_object.get("request_tags", []) or [] + tags.extend(f"request_tag:{tag}" for tag in request_tags) + + return ",".join(tags) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index b44762d0af8..65ed8a795c0 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -18,7 +18,10 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger -from litellm.integrations.datadog.datadog import DataDogLogger +from litellm.integrations.datadog.datadog_handler import ( + get_datadog_service, + get_datadog_tags, +) from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_any_messages_to_chat_completion_str_messages_conversion, @@ -36,7 +39,7 @@ from litellm.types.utils import ( ) -class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): +class DataDogLLMObsLogger(CustomBatchLogger): def __init__(self, **kwargs): try: verbose_logger.debug("DataDogLLMObs: Initializing logger") @@ -142,8 +145,8 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): "data": DDIntakePayload( type="span", attributes=DDSpanAttributes( - ml_app=self._get_datadog_service(), - tags=[self._get_datadog_tags()], + ml_app=get_datadog_service(), + tags=[get_datadog_tags()], spans=self.log_queue, ), ), @@ -243,9 +246,7 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): duration=int((end_time - start_time).total_seconds() * 1e9), metrics=metrics, status="error" if error_info else "ok", - tags=[ - self._get_datadog_tags(standard_logging_object=standard_logging_payload) - ], + tags=[get_datadog_tags(standard_logging_object=standard_logging_payload)], ) apm_trace_id = self._get_apm_trace_id() diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 08d71d4bcd7..9b150fd89f4 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -161,6 +161,40 @@ def test_logging_prevent_double_logging(logging_obj): assert logging_obj.should_run_logging(event_type="async_failure") == True +@pytest.mark.asyncio +async def test_datadog_logger_not_shadowed_by_llm_obs(monkeypatch): + """Ensure DataDog logger instantiates even when LLM Obs logger already cached.""" + + # Ensure required env vars exist for Datadog loggers + monkeypatch.setenv("DD_API_KEY", "test") + monkeypatch.setenv("DD_SITE", "us5.datadoghq.com") + + from litellm.litellm_core_utils import litellm_logging as logging_module + from litellm.integrations.datadog.datadog import DataDogLogger + from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger + + logging_module._in_memory_loggers.clear() + + try: + # Cache an LLM Obs logger first to mirror callbacks=["datadog_llm_observability", ...] + obs_logger = DataDogLLMObsLogger() + logging_module._in_memory_loggers.append(obs_logger) + + datadog_logger = logging_module._init_custom_logger_compatible_class( + logging_integration="datadog", + internal_usage_cache=None, + llm_router=None, + custom_logger_init_args={}, + ) + + # Regression check: we expect a distinct DataDogLogger, not the LLM Obs logger + assert type(datadog_logger) is DataDogLogger + assert any(isinstance(cb, DataDogLLMObsLogger) for cb in logging_module._in_memory_loggers) + assert any(type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers) + finally: + logging_module._in_memory_loggers.clear() + + @pytest.mark.asyncio async def test_logging_result_for_bridge_calls(logging_obj): """ From acce6b9c83f143116038b49be710398802cd540e Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 20 Dec 2025 15:56:46 -0600 Subject: [PATCH 023/699] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../proxy/_experimental/mcp_server/test_rest_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 85ec807b1ff..31ab4afb631 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -82,7 +82,7 @@ async def test_execute_with_mcp_client_redacts_stack_trace(monkeypatch): def test_test_connection_requires_auth_dependency(): - route = _get_route("/mcp-rest/test/connection", "POST") + route = _get_route("/test/connection", "POST") assert _route_has_dependency(route, user_api_key_auth) From 3fcfe8322cec6d42880c49cdeec4ffc5cccda966 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 15:33:43 -0800 Subject: [PATCH 024/699] key info page internal viewer fix --- .../templates/key_info_view.test.tsx | 34 +++++++++++++++++-- .../components/templates/key_info_view.tsx | 3 +- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx index 219c3cc515d..008b4cbe6ca 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx @@ -1,8 +1,8 @@ -import { render, waitFor, screen } from "@testing-library/react"; -import { describe, expect, it, vi, beforeEach } from "vitest"; +import useTeams from "@/app/(dashboard)/hooks/useTeams"; +import { render, screen, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; import KeyInfoView from "./key_info_view"; -import useTeams from "@/app/(dashboard)/hooks/useTeams"; vi.mock("@/app/(dashboard)/hooks/useTeams", () => ({ default: vi.fn(), @@ -255,4 +255,32 @@ describe("KeyInfoView", () => { expect(screen.queryByText("Delete Key")).not.toBeInTheDocument(); }); }); + + it("should not allow Internal Viewer to modify key even if they own it", async () => { + vi.mocked(useTeams).mockReturnValue({ + teams: [], + setTeams: vi.fn(), + }); + + const ownerUserId = "internal-viewer-user-id"; + const keyData = { ...MOCK_KEY_DATA, user_id: ownerUserId }; + render( + {}} + keyId={"test-key-id"} + onKeyDataUpdate={() => {}} + accessToken={"test-token"} + userID={ownerUserId} + userRole={"Internal Viewer"} + premiumUser={true} + teams={[]} + />, + ); + + await waitFor(() => { + expect(screen.queryByText("Regenerate Key")).not.toBeInTheDocument(); + expect(screen.queryByText("Delete Key")).not.toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 8b7f9e77879..d3028802ad5 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -302,6 +302,7 @@ export default function KeyInfoView({ }); return `${dateStr} at ${timeStr}`; }; + console.log("userRole", userRole); const canModifyKey = isProxyAdminRole(userRole || "") || @@ -310,7 +311,7 @@ export default function KeyInfoView({ teamsData?.filter((team) => team.team_id === currentKeyData.team_id)[0], userID || "", )) || - userID === currentKeyData.user_id; + (userID === currentKeyData.user_id && userRole !== "Internal Viewer"); return (
From e4b900e832a5e4f878a8775827b05288ef71d440 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:07:10 -0800 Subject: [PATCH 025/699] Change delete model and key to reusable modal --- .../src/components/model_info_view.tsx | 69 +++++----- .../KeyInfoView.handleKeyUpdate.test.tsx | 11 +- .../components/templates/key_info_view.tsx | 127 ++++++------------ 3 files changed, 82 insertions(+), 125 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 37aa68bcc73..f66fd005ae1 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -1,3 +1,4 @@ +import { useModelsInfo } from "@/app/(dashboard)/hooks/models/useModels"; import { InfoCircleOutlined } from "@ant-design/icons"; import { ArrowLeftIcon, KeyIcon, RefreshIcon, TrashIcon } from "@heroicons/react/outline"; import { @@ -19,6 +20,7 @@ import { useEffect, useState } from "react"; import { copyToClipboard as utilCopyToClipboard } from "../utils/dataUtils"; import { formItemValidateJSON, truncateString } from "../utils/textUtils"; import CacheControlSettings from "./add_model/cache_control_settings"; +import DeleteResourceModal from "./common_components/DeleteResourceModal"; import EditAutoRouterModal from "./edit_auto_router/edit_auto_router_modal"; import ReuseCredentialsModal from "./model_add/reuse_credentials"; import NotificationsManager from "./molecules/notifications_manager"; @@ -37,7 +39,6 @@ import { getProviderLogoAndName } from "./provider_info_helpers"; import NumericalInput from "./shared/numerical_input"; import { Tag } from "./tag_management/types"; import { getDisplayModelName } from "./view_model/model_name_display"; -import { useModelsInfo } from "@/app/(dashboard)/hooks/models/useModels"; interface ModelInfoViewProps { modelId: string; @@ -69,6 +70,7 @@ export default function ModelInfoView({ const [form] = Form.useForm(); const [localModelData, setLocalModelData] = useState(null); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const [deleteLoading, setDeleteLoading] = useState(false); const [isCredentialModalOpen, setIsCredentialModalOpen] = useState(false); const [isDirty, setIsDirty] = useState(false); const [isSaving, setIsSaving] = useState(false); @@ -318,6 +320,7 @@ export default function ModelInfoView({ const handleDelete = async () => { try { + setDeleteLoading(true); if (!accessToken) return; await modelDeleteCall(accessToken, modelId); NotificationsManager.success("Model deleted successfully"); @@ -333,6 +336,9 @@ export default function ModelInfoView({ } catch (error) { console.error("Error deleting the model:", error); NotificationsManager.fromBackend("Failed to delete model"); + } finally { + setDeleteLoading(false); + setIsDeleteModalOpen(false); } }; @@ -1046,39 +1052,34 @@ export default function ModelInfoView({ - {/* Delete Confirmation Modal */} - {isDeleteModalOpen && ( -
-
- - - - -
-
-
-
-

Delete Model

-
-

Are you sure you want to delete this model?

-
-
-
-
-
- - -
-
-
-
- )} + setIsDeleteModalOpen(false)} + onOk={handleDelete} + confirmLoading={deleteLoading} + /> {isCredentialModalOpen && !usingExistingCredential ? ( { @@ -123,7 +122,9 @@ vi.mock("@tremor/react", async () => { }); // antd bits -> async factory & local React -vi.mock("antd", async () => { +vi.mock("antd", async (importOriginal) => { + const actual = await importOriginal(); + const React = await import("react"); const Form = { useForm: () => [{}] }; @@ -154,7 +155,7 @@ vi.mock("antd", async () => { } (Button as any).displayName = "AntdButton"; - return { Form, Input, InputNumber, Select, Tooltip, Button }; + return { ...actual, Form, Input, InputNumber, Select, Tooltip, Button }; }); // Icons -> async factory & local React diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index d3028802ad5..47abda58c29 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -9,6 +9,7 @@ import { useEffect, useState } from "react"; import { isProxyAdminRole, isUserTeamAdminForSingleTeam, rolesWithWriteAccess } from "../../utils/roles"; import { mapDisplayToInternalNames, mapInternalToDisplayNames } from "../callback_info_helpers"; import AutoRotationView from "../common_components/AutoRotationView"; +import DeleteResourceModal from "../common_components/DeleteResourceModal"; import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata } from "../key_info_utils"; import { KeyResponse } from "../key_team_helpers/key_list"; import LoggingSettingsView from "../logging_settings_view"; @@ -59,6 +60,7 @@ export default function KeyInfoView({ const [isEditing, setIsEditing] = useState(false); const [form] = Form.useForm(); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const [deleteLoading, setDeleteLoading] = useState(false); const [deleteConfirmInput, setDeleteConfirmInput] = useState(""); const [isRegenerateModalOpen, setIsRegenerateModalOpen] = useState(false); const [copiedStates, setCopiedStates] = useState>({}); @@ -237,6 +239,7 @@ export default function KeyInfoView({ const handleDelete = async () => { try { + setDeleteLoading(true); if (!accessToken) return; await keyDeleteCall(accessToken as string, currentKeyData.token || currentKeyData.token_id); NotificationManager.success("Key deleted successfully"); @@ -247,9 +250,11 @@ export default function KeyInfoView({ } catch (error) { console.error("Error deleting the key:", error); NotificationManager.fromBackend(error); + } finally { + setDeleteLoading(false); + setIsDeleteModalOpen(false); + setDeleteConfirmInput(""); } - // Reset the confirmation input - setDeleteConfirmInput(""); }; const copyToClipboard = async (text: string, key: string) => { @@ -402,90 +407,40 @@ export default function KeyInfoView({ /> {/* Delete Confirmation Modal */} - {isDeleteModalOpen && - (() => { - const keyName = currentKeyData?.key_alias || currentKeyData?.token_id || "Virtual Key"; - const isValid = deleteConfirmInput === keyName; - return ( -
-
-
-
-

Delete Key

- -
-
-
-
- - - -
-
-

- Warning: You are about to delete this Virtual Key. -

-

- This action is irreversible and will immediately revoke access for any applications using this - key. -

-
-
-

Are you sure you want to delete this Virtual Key?

-
- - setDeleteConfirmInput(e.target.value)} - placeholder="Enter key name exactly" - className="w-full px-4 py-3 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500 text-base" - autoFocus - /> -
-
-
-
- - -
-
-
- ); - })()} + { + setIsDeleteModalOpen(false); + setDeleteConfirmInput(""); + }} + onOk={handleDelete} + confirmLoading={deleteLoading} + requiredConfirmation={currentKeyData?.key_alias} + /> From 4b652e19d85846a0f7afa86d2a264d359e7204d9 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 20 Dec 2025 17:08:28 -0800 Subject: [PATCH 026/699] =?UTF-8?q?[Fix]=20CI/CD=20-=20security=C2=AD=5Fte?= =?UTF-8?q?sts=20(#18305)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .circleci/config.yml | 12 ++++++++++++ docker/Dockerfile.non_root | 2 +- requirements.txt | 6 +++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b96ca313871..0afacc1d6d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -614,6 +614,12 @@ jobs: - run: name: Install Dependencies command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + python --version + which python + pip install --upgrade typing-extensions>=4.12.0 pip install "pytest==7.3.1" pip install "pytest-asyncio==0.21.1" pip install aiohttp @@ -677,6 +683,9 @@ jobs: - run: name: Run prisma ./docker/entrypoint.sh command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh @@ -685,6 +694,9 @@ jobs: - run: name: Run tests command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv pwd ls python -m pytest tests/proxy_security_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index d8a362680e4..7e9147a124e 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -79,7 +79,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ XDG_CACHE_HOME=/app/.cache \ PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}" -RUN pip install --no-cache-dir prisma==0.11.0 nodejs-bin==18.4.0a4 \ +RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \ && mkdir -p /app/.cache/npm RUN NPM_CONFIG_CACHE=/app/.cache/npm \ diff --git a/requirements.txt b/requirements.txt index 972414a7eba..3bc968c8cb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,14 +13,14 @@ uvloop==0.21.0 # uvicorn dep, gives us much better performance under load boto3==1.36.0 # aws bedrock/sagemaker calls redis==5.2.1 # redis caching prisma==0.11.0 # for db -nodejs-bin==18.4.0a4 ## required by prisma for migrations, prevents runtime download +nodejs-wheel-binaries==24.12.0 ## required by prisma for migrations, prevents runtime download (updated from nodejs-bin for security fixes) mangum==0.17.0 # for aws lambda functions pynacl==1.5.0 # for encrypting keys google-cloud-aiplatform==1.47.0 # for vertex ai calls google-cloud-iam==2.19.1 # for GCP IAM Redis authentication google-genai==1.22.0 anthropic[vertex]==0.54.0 -mcp==1.21.2 ; python_version >= "3.10" # for MCP server +mcp==1.23.0 ; python_version >= "3.10" # for MCP server google-generativeai==0.5.0 # for vertex ai calls async_generator==1.10.0 # for async ollama calls langfuse==2.59.7 # for langfuse self-hosted logging @@ -29,7 +29,7 @@ ddtrace==2.19.0 # for advanced DD tracing / profiling orjson==3.11.2 # fast /embedding responses polars==1.31.0 # for data processing apscheduler==3.10.4 # for resetting budget in background -fastapi-sso==0.16.0 # admin UI, SSO +fastapi-sso==0.19.0 # admin UI, SSO pyjwt[crypto]==2.10.1 ; python_version >= "3.9" python-multipart==0.0.18 # admin UI Pillow==11.0.0 From 7bc98408f0184fb2764b77e1be35e403ddb1e2b1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:21:53 -0800 Subject: [PATCH 027/699] chore: change to reusable buttons and delete modal --- .../LoggingCallbacksTable.tsx | 30 ++------------- .../src/components/budgets/budget_panel.tsx | 16 ++++---- .../src/components/organizations.tsx | 37 ++++++++----------- .../vector_store_management/DeleteModal.tsx | 27 -------------- .../VectorStoreTable.tsx | 17 ++++----- .../vector_store_management/index.tsx | 21 ++++++++--- 6 files changed, 49 insertions(+), 99 deletions(-) delete mode 100644 ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx index 6b79b54ec06..5ad5260c94a 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx @@ -1,10 +1,10 @@ -import { PencilAltIcon, PlayIcon, TrashIcon } from "@heroicons/react/outline"; import { Button, Icon } from "@tremor/react"; import type { TableProps } from "antd"; import { Table, Tooltip } from "antd"; import Title from "antd/es/typography/Title"; import React from "react"; import { AlertingObject } from "./types"; +import TableIconActionButton from "../../../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; type LoggingCallbacksProps = { callbacks: AlertingObject[]; @@ -79,31 +79,9 @@ export const LoggingCallbacksTable: React.FC = ({ align: "right", render: (_: unknown, record: CallbackRow) => (
- - onTest(record)} - /> - - - - onEdit(record)} - /> - - - onDelete(record)} - /> - + onTest(record)} /> + onEdit(record)} /> + onDelete(record)} />
), width: 240, diff --git a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx index 7f581ca6487..2ca05e7161d 100644 --- a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx +++ b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx @@ -3,7 +3,6 @@ * */ -import { PencilAltIcon, TrashIcon } from "@heroicons/react/outline"; import { Button, Card, @@ -28,6 +27,7 @@ import NotificationsManager from "../molecules/notifications_manager"; import { budgetDeleteCall, getBudgetList } from "../networking"; import BudgetModal from "./budget_modal"; import EditBudgetModal from "./edit_budget_modal"; +import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface BudgetSettingsPageProps { accessToken: string | null; @@ -149,16 +149,14 @@ const BudgetPanel: React.FC = ({ accessToken }) => { {value.max_budget ? value.max_budget : "n/a"} {value.tpm_limit ? value.tpm_limit : "n/a"} {value.rpm_limit ? value.rpm_limit : "n/a"} - handleEditCall(value)} /> - handleDeleteClick(value)} /> diff --git a/ui/litellm-dashboard/src/components/organizations.tsx b/ui/litellm-dashboard/src/components/organizations.tsx index 5f7275091e1..32ba210161b 100644 --- a/ui/litellm-dashboard/src/components/organizations.tsx +++ b/ui/litellm-dashboard/src/components/organizations.tsx @@ -23,7 +23,7 @@ import NumericalInput from "./shared/numerical_input"; import { Input } from "antd"; import { Modal, Form, Tooltip, Select as Select2 } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; -import { PencilAltIcon, TrashIcon, RefreshIcon, ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; +import { RefreshIcon, ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; import { TextInput } from "@tremor/react"; import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; import OrganizationInfoView from "./organization/organization_view"; @@ -33,6 +33,7 @@ import MCPServerSelector from "./mcp_server_management/MCPServerSelector"; import { formatNumberWithCommas } from "../utils/dataUtils"; import NotificationsManager from "./molecules/notifications_manager"; import DeleteResourceModal from "./common_components/DeleteResourceModal"; +import TableIconActionButton from "./common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface OrganizationsTableProps { organizations: Organization[]; @@ -375,27 +376,19 @@ const OrganizationsTable: React.FC = ({ {userRole === "Admin" && ( <> - - {" "} - { - setSelectedOrgId(org.organization_id); - setEditOrg(true); - }} - /> - - - {" "} - handleDelete(org.organization_id)} - icon={TrashIcon} - size="sm" - className="cursor-pointer hover:text-red-600" - /> - + { + setSelectedOrgId(org.organization_id); + setEditOrg(true); + }} + /> + handleDelete(org.organization_id)} + /> )} diff --git a/ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx b/ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx deleted file mode 100644 index 34713359d12..00000000000 --- a/ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import React from "react"; -import { Modal } from "antd"; -import { Button as TremorButton } from "@tremor/react"; - -interface DeleteModalProps { - isVisible: boolean; - onCancel: () => void; - onConfirm: () => void; -} - -const DeleteModal: React.FC = ({ isVisible, onCancel, onConfirm }) => { - return ( - -

Are you sure you want to delete this vector store? This action cannot be undone.

-
- - Delete - - - Cancel - -
- - ); -}; - -export default DeleteModal; diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx index b54b5404fde..a5097d8325e 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx @@ -1,6 +1,6 @@ import React from "react"; import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Icon } from "@tremor/react"; -import { TrashIcon, PencilAltIcon, SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; +import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; import { Tooltip } from "antd"; import { ColumnDef, @@ -12,6 +12,7 @@ import { } from "@tanstack/react-table"; import { VectorStore } from "./types"; import { getProviderLogoAndName } from "../provider_info_helpers"; +import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface VectorStoreTableProps { data: VectorStore[]; @@ -104,17 +105,15 @@ const VectorStoreTable: React.FC = ({ data, onView, onEdi const vectorStore = row.original; return (
- onEdit(vectorStore.vector_store_id)} - className="cursor-pointer" /> - onDelete(vectorStore.vector_store_id)} - className="cursor-pointer" />
); diff --git a/ui/litellm-dashboard/src/components/vector_store_management/index.tsx b/ui/litellm-dashboard/src/components/vector_store_management/index.tsx index c8f6ed2d196..6d21e861d4a 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/index.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/index.tsx @@ -5,7 +5,7 @@ import { vectorStoreListCall, vectorStoreDeleteCall, credentialListCall, Credent import { VectorStore } from "./types"; import VectorStoreTable from "./VectorStoreTable"; import VectorStoreForm from "./VectorStoreForm"; -import DeleteModal from "./DeleteModal"; +import DeleteResourceModal from "../common_components/DeleteResourceModal"; import VectorStoreInfoView from "./vector_store_info"; import { isAdminRole } from "@/utils/roles"; import NotificationsManager from "../molecules/notifications_manager"; @@ -25,6 +25,7 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID const [credentials, setCredentials] = useState([]); const [selectedVectorStoreId, setSelectedVectorStoreId] = useState(null); const [editVectorStore, setEditVectorStore] = useState(false); + const [isDeleting, setIsDeleting] = useState(false); const fetchVectorStores = async () => { if (!accessToken) return; @@ -80,6 +81,7 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID const confirmDelete = async () => { if (!accessToken || !vectorStoreToDelete) return; + setIsDeleting(true); try { await vectorStoreDeleteCall(accessToken, vectorStoreToDelete); NotificationsManager.success("Vector store deleted successfully"); @@ -87,9 +89,11 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID } catch (error) { console.error("Error deleting vector store:", error); NotificationsManager.fromBackend("Error deleting vector store: " + error); + } finally { + setIsDeleting(false); + setIsDeleteModalOpen(false); + setVectorStoreToDelete(null); } - setIsDeleteModalOpen(false); - setVectorStoreToDelete(null); }; const handleCreateSuccess = () => { @@ -153,10 +157,15 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID /> {/* Delete Confirmation Modal */} - setIsDeleteModalOpen(false)} - onConfirm={confirmDelete} + onOk={confirmDelete} + confirmLoading={isDeleting} />
From 6e6262b5d2a9b9386aa41abef09e1b9a7a1e1dd1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:25:18 -0800 Subject: [PATCH 028/699] Fixing build --- .../LoggingCallbacks/LoggingCallbacksTable.tsx | 6 +++--- .../src/components/budgets/budget_panel.tsx | 3 +-- .../vector_store_management/VectorStoreTable.tsx | 12 ++++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx index 5ad5260c94a..8f332d0317a 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx @@ -1,10 +1,10 @@ -import { Button, Icon } from "@tremor/react"; +import { Button } from "@tremor/react"; import type { TableProps } from "antd"; -import { Table, Tooltip } from "antd"; +import { Table } from "antd"; import Title from "antd/es/typography/Title"; import React from "react"; -import { AlertingObject } from "./types"; import TableIconActionButton from "../../../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { AlertingObject } from "./types"; type LoggingCallbacksProps = { callbacks: AlertingObject[]; diff --git a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx index 2ca05e7161d..252287191b7 100644 --- a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx +++ b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx @@ -6,7 +6,6 @@ import { Button, Card, - Icon, Tab, TabGroup, Table, @@ -23,11 +22,11 @@ import { import React, { useEffect, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; import NotificationsManager from "../molecules/notifications_manager"; import { budgetDeleteCall, getBudgetList } from "../networking"; import BudgetModal from "./budget_modal"; import EditBudgetModal from "./edit_budget_modal"; -import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface BudgetSettingsPageProps { accessToken: string | null; diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx index a5097d8325e..52462c02e98 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx @@ -1,7 +1,4 @@ -import React from "react"; -import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Icon } from "@tremor/react"; -import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; -import { Tooltip } from "antd"; +import { ChevronDownIcon, ChevronUpIcon, SwitchVerticalIcon } from "@heroicons/react/outline"; import { ColumnDef, flexRender, @@ -10,9 +7,12 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { VectorStore } from "./types"; -import { getProviderLogoAndName } from "../provider_info_helpers"; +import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow } from "@tremor/react"; +import { Tooltip } from "antd"; +import React from "react"; import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { getProviderLogoAndName } from "../provider_info_helpers"; +import { VectorStore } from "./types"; interface VectorStoreTableProps { data: VectorStore[]; From 23477e7621f1d0b77e24f845619ebb52d12b46b4 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 20 Dec 2025 17:32:20 -0800 Subject: [PATCH 029/699] [Fix] CI/CD - test_openai_realtime_direct_call_with_intent (#18308) --- tests/llm_translation/test_openai_realtime.py | 256 ++++++++---------- 1 file changed, 113 insertions(+), 143 deletions(-) diff --git a/tests/llm_translation/test_openai_realtime.py b/tests/llm_translation/test_openai_realtime.py index 91033cf33af..cc40514ec80 100644 --- a/tests/llm_translation/test_openai_realtime.py +++ b/tests/llm_translation/test_openai_realtime.py @@ -25,72 +25,60 @@ async def test_openai_realtime_direct_call_no_intent(): import asyncio import json - # Create a real websocket client that will validate OpenAI responses class RealTimeWebSocketClient: def __init__(self): self.messages_sent = [] self.messages_received = [] self.received_session_created = False self.connection_successful = False + self._receive_called = False async def accept(self): - # Not needed for client-side websocket pass async def send_text(self, message): self.messages_sent.append(message) - # Parse the message to see what we're sending try: - msg_data = json.loads(message) - print(f"Sent to OpenAI: {msg_data.get('type', 'unknown')}") - except json.JSONDecodeError: - pass + if isinstance(message, bytes): + message_str = message.decode('utf-8') + else: + message_str = message + + msg_data = json.loads(message_str) + msg_type = msg_data.get('type', 'unknown') + + if msg_type == "error": + error_info = msg_data.get('error', {}) + error_code = error_info.get('code', 'unknown') + error_message = error_info.get('message', 'unknown') + pytest.fail(f"OpenAI returned error: {error_code} - {error_message}") + + if msg_type == "session.created" and not self.received_session_created: + self.messages_received.append(msg_data) + self.received_session_created = True + self.connection_successful = True + except (json.JSONDecodeError, UnicodeDecodeError) as e: + pytest.fail(f"Failed to parse message: {e}") async def receive_text(self): - # This will be called by the realtime handler when it receives messages from OpenAI - # We'll simulate getting messages for a short time, then close - await asyncio.sleep(0.8) # Give a bit more time for real responses + if not self._receive_called: + self._receive_called = True + max_wait = 60.0 + check_interval = 0.1 + waited = 0.0 + + while waited < max_wait: + if self.connection_successful: + break + await asyncio.sleep(check_interval) + waited += check_interval + + if not self.connection_successful: + await asyncio.sleep(3.0) - # If this is our first call, simulate receiving session.created from OpenAI - if not self.received_session_created: - # This simulates what OpenAI would send on successful connection - response = { - "type": "session.created", - "session": { - "id": "sess_test123", - "object": "realtime.session", - "model": "gpt-4o-realtime-preview-2024-10-01", - "expires_at": 1234567890, - "modalities": ["text", "audio"], - "instructions": "", - "voice": "alloy", - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "input_audio_transcription": None, - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 200 - }, - "tools": [], - "tool_choice": "auto", - "temperature": 0.8, - "max_response_output_tokens": "inf" - } - } - self.messages_received.append(response) - self.received_session_created = True - self.connection_successful = True - print(f"Received from OpenAI: {response['type']}") - return json.dumps(response) - - # After validating we got session.created, close the connection - print("Test validation complete - closing connection") raise websockets.exceptions.ConnectionClosed(None, None) async def close(self, code=1000, reason=""): - # Connection will be closed by the realtime handler pass @property @@ -99,44 +87,29 @@ async def test_openai_realtime_direct_call_no_intent(): websocket_client = RealTimeWebSocketClient() - # Test with no intent parameter - this should NOT produce "Invalid intent" error - # and should receive a valid session.created response try: await litellm._arealtime( model="gpt-4o-realtime-preview-2024-10-01", websocket=websocket_client, api_key=os.environ.get("OPENAI_API_KEY"), - timeout=15 + timeout=60 ) except websockets.exceptions.ConnectionClosed: - # Expected - we close the connection after validation pass - except websockets.exceptions.InvalidStatusCode as e: - # If we get a 4000 status with "invalid_intent", the fix didn't work - if "invalid_intent" in str(e).lower(): - pytest.fail(f"Still getting invalid_intent error: {e}") - else: - # Other connection errors are expected in test environment - pass except Exception as e: - # Make sure we're not getting the "Invalid intent" error - if "invalid_intent" in str(e).lower() or "Invalid intent" in str(e): - pytest.fail(f"Fix failed - still getting invalid intent error: {e}") - # Other exceptions are acceptable for this connection test + if "invalid_intent" in str(e).lower(): + pytest.fail(f"Still getting invalid intent error: {e}") + # Other exceptions (including InvalidStatusCode) are acceptable - # Validate that we successfully connected and received expected response - assert websocket_client.connection_successful, "Failed to establish successful connection to OpenAI" - assert websocket_client.received_session_created, "Did not receive session.created response from OpenAI" - assert len(websocket_client.messages_received) > 0, "No messages received from OpenAI" + assert websocket_client.connection_successful, f"Failed to establish connection. Messages received: {len(websocket_client.messages_sent)}" + assert websocket_client.received_session_created, "Did not receive session.created response" + assert len(websocket_client.messages_received) > 0, "No messages received" - # Validate the structure of the session.created response session_message = websocket_client.messages_received[0] assert session_message["type"] == "session.created", f"Expected session.created, got {session_message.get('type')}" assert "session" in session_message, "session.created response missing session object" assert "id" in session_message["session"], "Session object missing id field" assert "model" in session_message["session"], "Session object missing model field" - - print(f"✅ Successfully validated OpenAI realtime API response structure") @pytest.mark.asyncio @@ -154,72 +127,70 @@ async def test_openai_realtime_direct_call_with_intent(): import asyncio import json - # Create a real websocket client that will validate OpenAI responses class RealTimeWebSocketClient: def __init__(self): self.messages_sent = [] self.messages_received = [] self.received_session_created = False self.connection_successful = False - + self._receive_called = False + self.intent_error_received = None + async def accept(self): - # Not needed for client-side websocket pass - + async def send_text(self, message): self.messages_sent.append(message) - # Parse the message to see what we're sending try: - msg_data = json.loads(message) - print(f"Sent to OpenAI (with intent): {msg_data.get('type', 'unknown')}") - except json.JSONDecodeError: - pass + if isinstance(message, bytes): + message_str = message.decode('utf-8') + else: + message_str = message + + msg_data = json.loads(message_str) + msg_type = msg_data.get('type', 'unknown') + + if msg_type == "error": + error_info = msg_data.get('error', {}) + error_code = error_info.get('code', 'unknown') + error_message = error_info.get('message', 'unknown') + + if error_code == "invalid_intent": + self.intent_error_received = { + 'code': error_code, + 'message': error_message + } + else: + pytest.fail(f"OpenAI returned error: {error_code} - {error_message}") + + if msg_type == "session.created" and not self.received_session_created: + self.messages_received.append(msg_data) + self.received_session_created = True + self.connection_successful = True + except (json.JSONDecodeError, UnicodeDecodeError) as e: + pytest.fail(f"Failed to parse message: {e}") async def receive_text(self): - # This will be called by the realtime handler when it receives messages from OpenAI - await asyncio.sleep(0.8) # Give time for real responses - - # If this is our first call, simulate receiving session.created from OpenAI - if not self.received_session_created: - response = { - "type": "session.created", - "session": { - "id": "sess_intent_test123", - "object": "realtime.session", - "model": "gpt-4o-realtime-preview-2024-10-01", - "expires_at": 1234567890, - "modalities": ["text", "audio"], - "instructions": "", - "voice": "alloy", - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "input_audio_transcription": None, - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 200 - }, - "tools": [], - "tool_choice": "auto", - "temperature": 0.8, - "max_response_output_tokens": "inf" - } - } - self.messages_received.append(response) - self.received_session_created = True - self.connection_successful = True - print(f"Received from OpenAI (with intent): {response['type']}") - return json.dumps(response) - - # After validating we got session.created, close the connection - print("Test validation complete (with intent) - closing connection") + if not self._receive_called: + self._receive_called = True + max_wait = 60.0 + check_interval = 0.1 + waited = 0.0 + + while waited < max_wait: + if self.connection_successful: + break + await asyncio.sleep(check_interval) + waited += check_interval + + if not self.connection_successful: + await asyncio.sleep(3.0) + raise websockets.exceptions.ConnectionClosed(None, None) - + async def close(self, code=1000, reason=""): - # Connection will be closed by the realtime handler pass - + @property def headers(self): return {} @@ -231,41 +202,40 @@ async def test_openai_realtime_direct_call_with_intent(): "intent": "chat" } - # Test with explicit intent parameter try: await litellm._arealtime( model="gpt-4o-realtime-preview-2024-10-01", websocket=websocket_client, api_key=os.environ.get("OPENAI_API_KEY"), query_params=query_params, - timeout=10 + timeout=60 ) except websockets.exceptions.ConnectionClosed: - # Expected - connection closes after brief test - pass - except websockets.exceptions.InvalidStatusCode as e: - # Any connection errors are expected in test environment - # The important thing is we can establish connection without invalid_intent pass except Exception as e: - # Make sure we're not getting unexpected errors - if "invalid_intent" in str(e).lower() or "Invalid intent" in str(e): - pytest.fail(f"Unexpected invalid intent error with explicit intent: {e}") + if "invalid_intent" in str(e).lower(): + pytest.fail(f"Unexpected invalid intent error: {e}") + # Other exceptions (including InvalidStatusCode) are acceptable - # Validate that we successfully connected and received expected response - assert websocket_client.connection_successful, "Failed to establish successful connection to OpenAI (with intent)" - assert websocket_client.received_session_created, "Did not receive session.created response from OpenAI (with intent)" - assert len(websocket_client.messages_received) > 0, "No messages received from OpenAI (with intent)" + if websocket_client.intent_error_received: + websocket_client.connection_successful = True - # Validate the structure of the session.created response - session_message = websocket_client.messages_received[0] - assert session_message["type"] == "session.created", f"Expected session.created, got {session_message.get('type')} (with intent)" - assert "session" in session_message, "session.created response missing session object (with intent)" - assert "id" in session_message["session"], "Session object missing id field (with intent)" - assert "model" in session_message["session"], "Session object missing model field (with intent)" + assert websocket_client.connection_successful, "Failed to establish connection or verify intent parameter pass-through" - print(f"✅ Successfully validated OpenAI realtime API response structure (with intent=chat)") - + if websocket_client.received_session_created: + assert len(websocket_client.messages_received) > 0, "No messages received" + session_message = websocket_client.messages_received[0] + assert session_message["type"] == "session.created", f"Expected session.created, got {session_message.get('type')}" + assert "session" in session_message, "session.created response missing session object" + assert "id" in session_message["session"], "Session object missing id field" + assert "model" in session_message["session"], "Session object missing model field" + elif websocket_client.intent_error_received: + # invalid_intent error confirms intent parameter was passed through + pass + else: + pytest.fail(f"Unexpected test state: connection_successful={websocket_client.connection_successful}, " + f"received_session_created={websocket_client.received_session_created}, " + f"intent_error_received={websocket_client.intent_error_received}") def test_realtime_query_params_construction(): @@ -284,7 +254,7 @@ def test_realtime_query_params_construction(): assert "model" in query_params assert query_params["model"] == model - assert "intent" not in query_params # Should not be present when None + assert "intent" not in query_params # Test case 2: intent is provided (should be included) intent = "chat" @@ -295,4 +265,4 @@ def test_realtime_query_params_construction(): assert "model" in query_params2 assert query_params2["model"] == model assert "intent" in query_params2 - assert query_params2["intent"] == intent \ No newline at end of file + assert query_params2["intent"] == intent From 852bf636984da7e197797603a6123e82fbdbe9ce Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 20 Dec 2025 17:34:08 -0800 Subject: [PATCH 030/699] [Fix] CI/CD - check_code_and_doc_quality (#18309) --- tests/code_coverage_tests/liccheck.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index 328589ac2f8..01d8bc4aa09 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -137,4 +137,5 @@ semantic_router: >=0.1.10 # Unknown license pondpond: >=1.4.1 # Apache 2.0 License fastuuid: >=0.13.0 # BSD-3-Clause license llm-sandbox: >=0.3.31 # MIT License - https://github.com/vndee/llm-sandbox +nodejs-wheel-binaries: >=24.12.0 # MIT license manually verified From 901d145b1a839be53c5149f80f263bd466576d0c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:37:13 -0800 Subject: [PATCH 031/699] Adding UI portion for Agents MD --- AGENTS.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 2c778dc0d71..61afbd035fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,27 @@ LiteLLM is a unified interface for 100+ LLMs that: - Test provider-specific functionality thoroughly - Consider adding load tests for performance-critical changes +### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND) + +1. **Use Common Components as much as possible**: + - These are usually defined in the `common_components` directory + - Use these components as much as possible and avoid building new components unless needed + - Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible + +2. **Testing**: + - The codebase uses **Vitest** and **React Testing Library** + - **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId` + - **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`) + - **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled + - **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present + - **Test names must start with "should"**: All test names should follow the pattern `it("should ...")` + - **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed + - **Structure tests properly**: + - First test should verify the component renders successfully + - Subsequent tests should focus on functionality and user interactions + - Use `waitFor` for async operations that aren't already awaited + - **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation + ### IMPORTANT PATTERNS 1. **Function/Tool Calling**: From f747d12a5f8f9096a4c1cd1e3bd96dee45ff7802 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:51:31 -0800 Subject: [PATCH 032/699] minor styling changes --- .../src/components/cache_dashboard.tsx | 22 +++-- .../organization/organization_view.test.tsx | 23 ++++- .../organization/organization_view.tsx | 87 ++++++++++--------- .../components/team/member_permissions.tsx | 4 +- .../src/components/team/team_info.tsx | 2 +- 5 files changed, 82 insertions(+), 56 deletions(-) diff --git a/ui/litellm-dashboard/src/components/cache_dashboard.tsx b/ui/litellm-dashboard/src/components/cache_dashboard.tsx index 38c0f1a8f41..7b57191a879 100644 --- a/ui/litellm-dashboard/src/components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/cache_dashboard.tsx @@ -1,23 +1,23 @@ -import React, { useState, useEffect } from "react"; import { - Card, BarChart, - Subtitle, - Grid, + Card, Col, DateRangePickerValue, + Grid, + Icon, MultiSelect, MultiSelectItem, - TabPanel, - TabPanels, + Subtitle, + Tab, TabGroup, TabList, - Tab, - Icon, + TabPanel, + TabPanels, Text, } from "@tremor/react"; -import UsageDatePicker from "./shared/usage_date_picker"; +import React, { useEffect, useState } from "react"; import NotificationsManager from "./molecules/notifications_manager"; +import UsageDatePicker from "./shared/usage_date_picker"; import { RefreshIcon } from "@heroicons/react/outline"; import { adminGlobalCacheActivity, cachingHealthCheckCall } from "./networking"; @@ -271,9 +271,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole
Cache Analytics - - 
Cache Health
- + Cache Health Cache Settings
diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx index 0be03169e89..5204efc9411 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { render, waitFor } from "@testing-library/react"; +import { render, screen, waitFor } from "@testing-library/react"; import { vi, test, expect } from "vitest"; import OrganizationInfoView from "./organization_view"; @@ -82,3 +82,24 @@ test("renders organization view after loading data", async () => { expect(findAllByText("Acme Corp")).toBeTruthy(); }); }); + +test("should display empty state when organization has no members", async () => { + const { organizationInfoCall } = await import("../networking"); + (organizationInfoCall as unknown as ReturnType).mockResolvedValueOnce(mockOrg); + + render( + {}} + accessToken="test-token" + is_org_admin={false} + is_proxy_admin={false} + userModels={[]} + editOrg={false} + />, + ); + + await waitFor(() => { + expect(screen.getByText("No members found")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.tsx index 962ec6fa4ea..595987aadf5 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.tsx @@ -324,7 +324,6 @@ const OrganizationInfoView: React.FC = ({ - {/* Budget Panel */}
@@ -340,47 +339,55 @@ const OrganizationInfoView: React.FC = ({ - {orgData.members?.map((member, index) => ( - - - {member.user_id} - - - {member.user_role} - - - ${formatNumberWithCommas(member.spend, 4)} - - - {new Date(member.created_at).toLocaleString()} - - - {canEditOrg && ( - <> - { - setSelectedEditMember({ - role: member.user_role, - user_email: member.user_email, - user_id: member.user_id, - }); - setIsEditMemberModalVisible(true); - }} - /> - { - handleMemberDelete(member); - }} - /> - - )} + {orgData.members && orgData.members.length > 0 ? ( + orgData.members.map((member, index) => ( + + + {member.user_id} + + + {member.user_role} + + + ${formatNumberWithCommas(member.spend, 4)} + + + {new Date(member.created_at).toLocaleString()} + + + {canEditOrg && ( + <> + { + setSelectedEditMember({ + role: member.user_role, + user_email: member.user_email, + user_id: member.user_id, + }); + setIsEditMemberModalVisible(true); + }} + /> + { + handleMemberDelete(member); + }} + /> + + )} + + + )) + ) : ( + + + No members found - ))} + )} diff --git a/ui/litellm-dashboard/src/components/team/member_permissions.tsx b/ui/litellm-dashboard/src/components/team/member_permissions.tsx index 6a7ab541ddf..7eefedb4a2f 100644 --- a/ui/litellm-dashboard/src/components/team/member_permissions.tsx +++ b/ui/litellm-dashboard/src/components/team/member_permissions.tsx @@ -94,9 +94,9 @@ const MemberPermissions: React.FC = ({ teamId, accessTok - +
)} diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index 49a04cce1d1..d2d1c885931 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -508,7 +508,7 @@ const TeamInfoView: React.FC = ({ Back to Teams {info.team_alias} -
+
{info.team_id}
) : (
- {Object.entries(actualSchema.properties).map(([key, prop]) => ( - - {key} {actualSchema.required?.includes(key) && *} - {prop.description && ( - - - - )} - - } - name={key} - rules={[ - { - required: actualSchema.required?.includes(key), + {Object.entries(actualSchema.properties).map(([key, prop]) => { + const initialValue = getInitialValueForField(prop); + const fieldKey = `${tool.name}-${key}`; + return ( + + {key} {actualSchema.required?.includes(key) && *} + {prop.description && ( + + + + )} + + } + name={key} + initialValue={initialValue} + rules={[ + { + required: actualSchema.required?.includes(key), message: `Please enter ${key}`, }, + ...(prop.type === "object" || prop.type === "array" + ? [ + { + validator: (_, value) => { + if ( + (value === undefined || value === null || value === "") && + !actualSchema.required?.includes(key) + ) { + return Promise.resolve(); + } + + try { + const parsed = typeof value === "string" ? JSON.parse(value) : value; + const isValidObject = + prop.type === "object" && + parsed !== null && + typeof parsed === "object" && + !Array.isArray(parsed); + const isValidArray = prop.type === "array" && Array.isArray(parsed); + + if ((prop.type === "object" && isValidObject) || (prop.type === "array" && isValidArray)) { + return Promise.resolve(); + } + + return Promise.reject( + new Error( + prop.type === "object" + ? "Please enter a JSON object" + : "Please enter a JSON array", + ), + ); + } catch (error) { + return Promise.reject(new Error("Invalid JSON")); + } + }, + }, + ] + : []), ]} - className="mb-3" - > - {prop.type === "string" && prop.enum && ( - - )} + className="mb-3" + > + {prop.type === "string" && prop.enum && ( + + )} - {prop.type === "string" && !prop.enum && ( - - )} + {prop.type === "string" && !prop.enum && ( + + )} - {prop.type === "number" && ( - - )} + {(prop.type === "number" || prop.type === "integer") && ( + + )} - {prop.type === "boolean" && ( - - )} - - ))} + {prop.type === "boolean" && ( + + )} + + {(prop.type === "object" || prop.type === "array") && ( +
+