From 7b6a00d3df82352ae1cdb76f6e7a5c6ac632b7e6 Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 16 Dec 2025 13:06:33 -0300 Subject: [PATCH 001/131] fix: remove deprecated Groq models and update model registry - Remove 20 deprecated/unavailable Groq models from registry - Add groq/meta-llama/llama-guard-4-12b (new safety model) - Add supports_vision to Llama 4 models (maverick, scout) - Update Groq documentation with current model list - Clean up test file references to deprecated models Fixes #18043 --- docs/my-website/docs/providers/groq.md | 31 +-- ...odel_prices_and_context_window_backup.json | 240 +----------------- model_prices_and_context_window.json | 240 +----------------- tests/test_litellm/test_utils.py | 2 - 4 files changed, 24 insertions(+), 489 deletions(-) diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md index ebed31f720f..55c222635d2 100644 --- a/docs/my-website/docs/providers/groq.md +++ b/docs/my-website/docs/providers/groq.md @@ -150,15 +150,15 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion | Model Name | Usage | |--------------------|---------------------------------------------------------| -| llama-3.1-8b-instant | `completion(model="groq/llama-3.1-8b-instant", messages)` | -| llama-3.1-70b-versatile | `completion(model="groq/llama-3.1-70b-versatile", messages)` | -| llama3-8b-8192 | `completion(model="groq/llama3-8b-8192", messages)` | -| llama3-70b-8192 | `completion(model="groq/llama3-70b-8192", messages)` | -| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` | -| mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` | -| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` | -| moonshotai/kimi-k2-instruct | `completion(model="groq/moonshotai/kimi-k2-instruct", messages)` | -| qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` | +| llama-3.3-70b-versatile | `completion(model="groq/llama-3.3-70b-versatile", messages)` | +| llama-3.1-8b-instant | `completion(model="groq/llama-3.1-8b-instant", messages)` | +| meta-llama/llama-4-scout-17b-16e-instruct | `completion(model="groq/meta-llama/llama-4-scout-17b-16e-instruct", messages)` | +| meta-llama/llama-4-maverick-17b-128e-instruct | `completion(model="groq/meta-llama/llama-4-maverick-17b-128e-instruct", messages)` | +| meta-llama/llama-guard-4-12b | `completion(model="groq/meta-llama/llama-guard-4-12b", messages)` | +| qwen/qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` | +| moonshotai/kimi-k2-instruct-0905 | `completion(model="groq/moonshotai/kimi-k2-instruct-0905", messages)` | +| openai/gpt-oss-120b | `completion(model="groq/openai/gpt-oss-120b", messages)` | +| openai/gpt-oss-20b | `completion(model="groq/openai/gpt-oss-20b", messages)` | ## Groq - Tool / Function Calling Example @@ -261,31 +261,28 @@ if tool_calls: print("second response\n", second_response) ``` -## Groq - Vision Example +## Groq - Vision Example -Select Groq models support vision. Check out their [model list](https://console.groq.com/docs/vision) for more details. +Groq's Llama 4 models support vision. Check out their [model list](https://console.groq.com/docs/vision) for more details. ```python -from litellm import completion - -import os +import os from litellm import completion os.environ["GROQ_API_KEY"] = "your-api-key" -# openai call response = completion( - model = "groq/llama-3.2-11b-vision-preview", + model = "groq/meta-llama/llama-4-scout-17b-16e-instruct", messages=[ { "role": "user", "content": [ { "type": "text", - "text": "What’s in this image?" + "text": "What's in this image?" }, { "type": "image_url", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2af548ce07e..5754899846c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -17509,75 +17509,6 @@ "supports_response_schema": true, "supports_vision": true }, - "groq/deepseek-r1-distill-llama-70b": { - "input_cost_per_token": 7.5e-07, - "litellm_provider": "groq", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 9.9e-07, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/distil-whisper-large-v3-en": { - "input_cost_per_second": 5.56e-06, - "litellm_provider": "groq", - "mode": "audio_transcription", - "output_cost_per_second": 0.0 - }, - "groq/gemma-7b-it": { - "deprecation_date": "2024-12-18", - "input_cost_per_token": 7e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/gemma2-9b-it": { - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_function_calling": false, - "supports_response_schema": false, - "supports_tool_choice": false - }, - "groq/llama-3.1-405b-reasoning": { - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.1-70b-versatile": { - "deprecation_date": "2025-01-24", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/llama-3.1-8b-instant": { "input_cost_per_token": 5e-08, "litellm_provider": "groq", @@ -17590,97 +17521,6 @@ "supports_response_schema": false, "supports_tool_choice": true }, - "groq/llama-3.2-11b-text-preview": { - "deprecation_date": "2024-10-28", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-11b-vision-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/llama-3.2-1b-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 4e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-3b-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 6e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-90b-text-preview": { - "deprecation_date": "2024-11-25", - "input_cost_per_token": 9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-90b-vision-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/llama-3.3-70b-specdec": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9.9e-07, - "supports_tool_choice": true - }, "groq/llama-3.3-70b-versatile": { "input_cost_per_token": 5.9e-07, "litellm_provider": "groq", @@ -17693,7 +17533,7 @@ "supports_response_schema": false, "supports_tool_choice": true }, - "groq/llama-guard-3-8b": { + "groq/meta-llama/llama-guard-4-12b": { "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, @@ -17702,44 +17542,6 @@ "mode": "chat", "output_cost_per_token": 2e-07 }, - "groq/llama2-70b-4096": { - "input_cost_per_token": 7e-07, - "litellm_provider": "groq", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama3-groq-70b-8192-tool-use-preview": { - "deprecation_date": "2025-01-06", - "input_cost_per_token": 8.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama3-groq-8b-8192-tool-use-preview": { - "deprecation_date": "2025-01-06", - "input_cost_per_token": 1.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "groq", @@ -17750,7 +17552,8 @@ "output_cost_per_token": 6e-07, "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "groq/meta-llama/llama-4-scout-17b-16e-instruct": { "input_cost_per_token": 1.1e-07, @@ -17762,41 +17565,8 @@ "output_cost_per_token": 3.4e-07, "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true - }, - "groq/mistral-saba-24b": { - "input_cost_per_token": 7.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 32000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.9e-07 - }, - "groq/mixtral-8x7b-32768": { - "deprecation_date": "2025-03-20", - "input_cost_per_token": 2.4e-07, - "litellm_provider": "groq", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 2.4e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/moonshotai/kimi-k2-instruct": { - "input_cost_per_token": 1e-06, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 16384, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 3e-06, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "groq/moonshotai/kimi-k2-instruct-0905": { "input_cost_per_token": 1e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2af548ce07e..5754899846c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -17509,75 +17509,6 @@ "supports_response_schema": true, "supports_vision": true }, - "groq/deepseek-r1-distill-llama-70b": { - "input_cost_per_token": 7.5e-07, - "litellm_provider": "groq", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 9.9e-07, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/distil-whisper-large-v3-en": { - "input_cost_per_second": 5.56e-06, - "litellm_provider": "groq", - "mode": "audio_transcription", - "output_cost_per_second": 0.0 - }, - "groq/gemma-7b-it": { - "deprecation_date": "2024-12-18", - "input_cost_per_token": 7e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/gemma2-9b-it": { - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_function_calling": false, - "supports_response_schema": false, - "supports_tool_choice": false - }, - "groq/llama-3.1-405b-reasoning": { - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.1-70b-versatile": { - "deprecation_date": "2025-01-24", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/llama-3.1-8b-instant": { "input_cost_per_token": 5e-08, "litellm_provider": "groq", @@ -17590,97 +17521,6 @@ "supports_response_schema": false, "supports_tool_choice": true }, - "groq/llama-3.2-11b-text-preview": { - "deprecation_date": "2024-10-28", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-11b-vision-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/llama-3.2-1b-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 4e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-3b-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 6e-08, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-90b-text-preview": { - "deprecation_date": "2024-11-25", - "input_cost_per_token": 9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.2-90b-vision-preview": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/llama-3.3-70b-specdec": { - "deprecation_date": "2025-04-14", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9.9e-07, - "supports_tool_choice": true - }, "groq/llama-3.3-70b-versatile": { "input_cost_per_token": 5.9e-07, "litellm_provider": "groq", @@ -17693,7 +17533,7 @@ "supports_response_schema": false, "supports_tool_choice": true }, - "groq/llama-guard-3-8b": { + "groq/meta-llama/llama-guard-4-12b": { "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, @@ -17702,44 +17542,6 @@ "mode": "chat", "output_cost_per_token": 2e-07 }, - "groq/llama2-70b-4096": { - "input_cost_per_token": 7e-07, - "litellm_provider": "groq", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 8e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama3-groq-70b-8192-tool-use-preview": { - "deprecation_date": "2025-01-06", - "input_cost_per_token": 8.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama3-groq-8b-8192-tool-use-preview": { - "deprecation_date": "2025-01-06", - "input_cost_per_token": 1.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "groq", @@ -17750,7 +17552,8 @@ "output_cost_per_token": 6e-07, "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "groq/meta-llama/llama-4-scout-17b-16e-instruct": { "input_cost_per_token": 1.1e-07, @@ -17762,41 +17565,8 @@ "output_cost_per_token": 3.4e-07, "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true - }, - "groq/mistral-saba-24b": { - "input_cost_per_token": 7.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 32000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.9e-07 - }, - "groq/mixtral-8x7b-32768": { - "deprecation_date": "2025-03-20", - "input_cost_per_token": 2.4e-07, - "litellm_provider": "groq", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 2.4e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/moonshotai/kimi-k2-instruct": { - "input_cost_per_token": 1e-06, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 16384, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 3e-06, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "groq/moonshotai/kimi-k2-instruct-0905": { "input_cost_per_token": 1e-06, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 2bd94488ba2..a3dc6f5085d 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -870,8 +870,6 @@ SKIP_MODELS = [ "jamba", "deepinfra", "mistral.", - "groq/llama-guard-3-8b", - "groq/gemma2-9b-it", ] # Bedrock models to block - organized by type From 9274860aa28c5d69d696077b92283dfcad90e0a5 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 14:54:26 -0800 Subject: [PATCH 002/131] Base commit --- litellm/proxy/management_endpoints/ui_sso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index d1db21a2706..5094fc5de97 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -594,7 +594,7 @@ def _build_sso_user_update_data( user_id: Optional[str], ) -> dict: """ - Build the update data dictionary for SSO user upsert. + Build the update data dictionary for SSO user upsert Args: result: The SSO response containing user information From 42d7d757a3bb1e5da02ed53c494ce6abca2f7e61 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 15:48:26 -0800 Subject: [PATCH 003/131] Adding role mappings to SSOConfig DB --- litellm/proxy/management_endpoints/ui_sso.py | 2 +- litellm/proxy/proxy_server.py | 1 + .../proxy_setting_endpoints.py | 12 ++ .../proxy/management_endpoints/ui_sso.py | 34 ++++- .../test_proxy_setting_endpoints.py | 135 ++++++++++++++++++ 5 files changed, 182 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 5094fc5de97..d1db21a2706 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -594,7 +594,7 @@ def _build_sso_user_update_data( user_id: Optional[str], ) -> dict: """ - Build the update data dictionary for SSO user upsert + Build the update data dictionary for SSO user upsert. Args: result: The SSO response containing user information diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8fdea95d7ad..dfadab1d531 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3578,6 +3578,7 @@ class ProxyConfig: ) if sso_settings is not None: # Capitalize all keys in sso_settings dictionary + sso_settings.sso_settings.pop("role_mappings", None) uppercase_sso_settings = { key.upper(): value for key, value in sso_settings.sso_settings.items() diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 9c99b625e9f..d9a41d38b22 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -433,10 +433,21 @@ async def get_sso_settings(): if sso_db_record and sso_db_record.sso_settings: # Load settings from database sso_settings_dict = dict(sso_db_record.sso_settings) + + # Extract role_mappings before removing it (it's a dict, not an env variable) + role_mappings_data = sso_settings_dict.pop("role_mappings", None) + role_mappings = None + if role_mappings_data: + from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings + if isinstance(role_mappings_data, dict): + role_mappings = RoleMappings(**role_mappings_data) + elif isinstance(role_mappings_data, RoleMappings): + role_mappings = role_mappings_data decrypted_sso_settings_dict = proxy_config._decrypt_and_set_db_env_variables(environment_variables=sso_settings_dict) # Build SSO config with database values or environment fallback + sso_config = SSOConfig( google_client_id=decrypted_sso_settings_dict.get("google_client_id", None), google_client_secret=decrypted_sso_settings_dict.get("google_client_secret", None), @@ -451,6 +462,7 @@ async def get_sso_settings(): proxy_base_url=decrypted_sso_settings_dict.get("proxy_base_url", None), user_email=decrypted_sso_settings_dict.get("user_email"), ui_access_mode=decrypted_sso_settings_dict.get("ui_access_mode"), + role_mappings=role_mappings, ) # Get the schema for UI display diff --git a/litellm/types/proxy/management_endpoints/ui_sso.py b/litellm/types/proxy/management_endpoints/ui_sso.py index 820b0164400..187d8c97c05 100644 --- a/litellm/types/proxy/management_endpoints/ui_sso.py +++ b/litellm/types/proxy/management_endpoints/ui_sso.py @@ -1,10 +1,12 @@ -from typing import List, Literal, Optional, Union +from typing import Dict, List, Literal, Optional, Union from pydantic import Field from typing_extensions import TypedDict from litellm.types.utils import LiteLLMPydanticObjectBase +from litellm.proxy._types import LitellmUserRoles + class LiteLLM_UpperboundKeyGenerateParams(LiteLLMPydanticObjectBase): """ @@ -60,6 +62,30 @@ class AccessControl_UI_AccessMode(LiteLLMPydanticObjectBase): sso_group_jwt_field: str +class RoleMappings(LiteLLMPydanticObjectBase): + """ + Configuration for mapping SSO groups to LiteLLM roles. + + The system will look at the group_claim field in the SSO token to determine + which role to assign the user based on the roles mapping. + """ + + provider: str = Field( + description="SSO Provider name (e.g., 'google', 'microsoft', 'generic')" + ) + group_claim: str = Field( + description="The field name in the SSO token that contains the groups array (e.g., 'groups', 'roles')" + ) + default_role: Optional[LitellmUserRoles] = Field( + default=None, + description="Default role to assign if user's groups don't match any role mappings. Must be a valid LitellmUserRoles value (e.g., 'proxy_admin', 'internal_user', 'proxy_admin_viewer')" + ) + roles: Dict[LitellmUserRoles, List[str]] = Field( + default_factory=dict, + description="Mapping of LiteLLM role names to arrays of SSO group names. Example: {'proxy_admin': ['group-1', 'group-2'], 'proxy_admin_viewer': ['group-3']}" + ) + + class SSOConfig(LiteLLMPydanticObjectBase): """ Configuration for SSO environment variables and settings @@ -127,6 +153,12 @@ class SSOConfig(LiteLLMPydanticObjectBase): description="Access mode for the UI", ) + # Role Mappings + role_mappings: Optional[RoleMappings] = Field( + default=None, + description="Configuration for mapping SSO groups to LiteLLM roles based on group claims in the SSO token", + ) + class DefaultTeamSSOParams(LiteLLMPydanticObjectBase): """ diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index d3c99151195..8fdfd6897a8 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -290,6 +290,10 @@ class TestProxySettingEndpoints: assert "google_client_id" in data["field_schema"]["properties"] assert "description" in data["field_schema"]["properties"]["google_client_id"] + # Verify role_mappings is present in response (can be None if not set) + assert "role_mappings" in values + assert values["role_mappings"] is None + # Verify find_unique was called with correct parameters mock_prisma.db.litellm_ssoconfig.find_unique.assert_called_once() call_args = mock_prisma.db.litellm_ssoconfig.find_unique.call_args @@ -863,6 +867,10 @@ class TestProxySettingEndpoints: assert values["google_client_secret"] == "decrypted_google_secret" assert values["microsoft_client_id"] == "decrypted_microsoft_id" assert values["proxy_base_url"] == "https://decrypted.example.com" + + # Verify role_mappings is present in response (can be None if not set) + assert "role_mappings" in values + assert values["role_mappings"] is None def test_update_sso_settings_to_database(self, mock_proxy_config, mock_auth, monkeypatch): """Test updating SSO settings saves to the dedicated database table""" @@ -1062,6 +1070,7 @@ class TestProxySettingEndpoints: assert values.get("google_client_id") is None assert values.get("google_client_secret") is None assert values.get("microsoft_client_id") is None + assert values.get("role_mappings") is None def test_update_sso_settings_no_database_connection(self, mock_proxy_config, mock_auth, monkeypatch): """Test updating SSO settings when database is not connected""" @@ -1088,3 +1097,129 @@ class TestProxySettingEndpoints: data = response.json() assert "error" in data["detail"] assert "Database not connected" in data["detail"]["error"] + + def test_get_sso_settings_with_role_mappings(self, mock_proxy_config, mock_auth, monkeypatch): + """Test getting SSO settings when role_mappings is present in database""" + from unittest.mock import AsyncMock, MagicMock + from litellm.proxy._types import LitellmUserRoles + + # Mock the prisma client with database record containing role_mappings + mock_prisma = MagicMock() + mock_db_record = MagicMock() + mock_db_record.sso_settings = { + "google_client_id": "test_google_client_id", + "role_mappings": { + "provider": "google", + "group_claim": "groups", + "default_role": LitellmUserRoles.INTERNAL_USER, + "roles": { + LitellmUserRoles.PROXY_ADMIN: ["admin-group"], + }, + }, + } + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=mock_db_record) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + # Mock decryption to return the values as-is (role_mappings should not be passed to decryption) + from litellm.proxy.proxy_server import proxy_config + def mock_decrypt(environment_variables): + # role_mappings should not be in environment_variables since it's extracted before decryption + assert "role_mappings" not in environment_variables + return environment_variables + + monkeypatch.setattr( + proxy_config, "_decrypt_and_set_db_env_variables", mock_decrypt + ) + + response = client.get("/get/sso_settings") + + assert response.status_code == 200 + data = response.json() + + # Verify role_mappings is returned correctly + values = data["values"] + assert "role_mappings" in values + assert values["role_mappings"] is not None + assert values["role_mappings"]["provider"] == "google" + assert values["role_mappings"]["group_claim"] == "groups" + assert values["role_mappings"]["default_role"] == LitellmUserRoles.INTERNAL_USER + assert values["role_mappings"]["roles"][LitellmUserRoles.PROXY_ADMIN] == ["admin-group"] + + def test_role_mappings_stored_and_retrieved(self, mock_proxy_config, mock_auth, monkeypatch): + """Test that role_mappings is properly stored and retrieved from SSO settings""" + import json + from unittest.mock import AsyncMock, MagicMock + from litellm.proxy._types import LitellmUserRoles + + monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key") + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + + # Mock the prisma client + mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() + mock_prisma.db.litellm_config = MagicMock() + mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + mock_prisma.db.litellm_config.update = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + # Mock encryption to return values as-is + from litellm.proxy.proxy_server import proxy_config + monkeypatch.setattr(proxy_config, "_encrypt_env_variables", lambda environment_variables: environment_variables) + + # SSO settings with role_mappings + role_mappings_data = { + "provider": "google", + "group_claim": "groups", + "default_role": LitellmUserRoles.INTERNAL_USER, + "roles": { + LitellmUserRoles.PROXY_ADMIN: ["admin-group"], + LitellmUserRoles.INTERNAL_USER: ["user-group"], + }, + } + + new_sso_settings = { + "google_client_id": "test_google_id", + "role_mappings": role_mappings_data, + } + + response = client.patch("/update/sso_settings", json=new_sso_settings) + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert "role_mappings" in data["settings"] + + # Verify role_mappings structure in response + returned_role_mappings = data["settings"]["role_mappings"] + assert returned_role_mappings["provider"] == "google" + assert returned_role_mappings["group_claim"] == "groups" + assert returned_role_mappings["default_role"] == LitellmUserRoles.INTERNAL_USER + assert returned_role_mappings["roles"][LitellmUserRoles.PROXY_ADMIN] == ["admin-group"] + + # Verify upsert was called with role_mappings in the data + assert mock_prisma.db.litellm_ssoconfig.upsert.called + call_args = mock_prisma.db.litellm_ssoconfig.upsert.call_args + create_data = call_args.kwargs["data"]["create"] + stored_sso_settings = json.loads(create_data["sso_settings"]) + assert "role_mappings" in stored_sso_settings + assert stored_sso_settings["role_mappings"]["provider"] == "google" + + # Now test retrieving role_mappings + mock_db_record = MagicMock() + mock_db_record.sso_settings = stored_sso_settings + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=mock_db_record) + monkeypatch.setattr( + proxy_config, "_decrypt_and_set_db_env_variables", lambda environment_variables: environment_variables + ) + + get_response = client.get("/get/sso_settings") + assert get_response.status_code == 200 + get_data = get_response.json() + + # Verify role_mappings is returned correctly + assert "role_mappings" in get_data["values"] + retrieved_role_mappings = get_data["values"]["role_mappings"] + assert retrieved_role_mappings is not None + assert retrieved_role_mappings["provider"] == "google" + assert retrieved_role_mappings["group_claim"] == "groups" + assert retrieved_role_mappings["default_role"] == LitellmUserRoles.INTERNAL_USER From 58330f852d82bd42f02f2e920cdb14ae0b927024 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 17:36:08 -0800 Subject: [PATCH 004/131] WIP waiting for okta --- litellm/proxy/management_endpoints/ui_sso.py | 134 ++++++++++++++++++- 1 file changed, 127 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index d1db21a2706..7a404bcda81 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -85,6 +85,58 @@ else: router = APIRouter() +def determine_role_from_groups( + user_groups: List[str], + role_mappings: "RoleMappings", +) -> Optional[LitellmUserRoles]: + """ + Determine the highest privilege role for a user based on their groups. + + Role hierarchy (highest to lowest): + - proxy_admin + - proxy_admin_viewer + - internal_user + - internal_user_viewer + + Args: + user_groups: List of group names from the SSO token + role_mappings: RoleMappings configuration object + + Returns: + The highest privilege role found, or default_role if no matches, or None + """ + if not role_mappings.roles: + # No role mappings configured, return default_role + return role_mappings.default_role + + # Role hierarchy (highest to lowest) + role_hierarchy = [ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] + + # Convert user_groups to a set for efficient lookup + user_groups_set = set(user_groups) if isinstance(user_groups, list) else set() + + # Find the highest privilege role the user belongs to + for role in role_hierarchy: + if role in role_mappings.roles: + role_groups = role_mappings.roles[role] + if isinstance(role_groups, list) and user_groups_set.intersection(set(role_groups)): + verbose_proxy_logger.debug( + f"User groups {user_groups} matched role '{role.value}' via groups: {role_groups}" + ) + return role + + # No matching groups found, return default_role + verbose_proxy_logger.debug( + f"User groups {user_groups} did not match any role mappings, using default_role: {role_mappings.default_role}" + ) + return role_mappings.default_role + + def process_sso_jwt_access_token( access_token_str: Optional[str], sso_jwt_handler: Optional[JWTHandler], @@ -243,6 +295,7 @@ def generic_response_convertor( response, jwt_handler: JWTHandler, sso_jwt_handler: Optional[JWTHandler] = None, + role_mappings: Optional["RoleMappings"] = None, ) -> CustomOpenID: generic_user_id_attribute_name = os.getenv( "GENERIC_USER_ID_ATTRIBUTE", "preferred_username" @@ -281,16 +334,48 @@ def generic_response_convertor( team_ids = jwt_handler.get_team_ids_from_jwt(cast(dict, response)) all_teams.extend(team_ids) - # Extract user role from SSO response - user_role_from_sso = get_nested_value(response, generic_user_role_attribute_name) + # Determine user role based on role_mappings if available + # Only apply role_mappings for GENERIC SSO provider user_role: Optional[LitellmUserRoles] = None - if user_role_from_sso is not None: - role = get_litellm_user_role(user_role_from_sso) - if role is not None: - user_role = role + + if role_mappings is not None and role_mappings.provider.lower() in ["generic", "okta"]: + # Use role_mappings to determine role from groups + group_claim = role_mappings.group_claim + user_groups_raw = get_nested_value(response, group_claim) + + # Handle different formats: could be a list, string (comma-separated), or single value + user_groups: List[str] = [] + if isinstance(user_groups_raw, list): + user_groups = [str(g) for g in user_groups_raw] + elif isinstance(user_groups_raw, str): + # Handle comma-separated string + user_groups = [g.strip() for g in user_groups_raw.split(",") if g.strip()] + elif user_groups_raw is not None: + # Single value + user_groups = [str(user_groups_raw)] + + if user_groups: + user_role = determine_role_from_groups(user_groups, role_mappings) verbose_proxy_logger.debug( - f"Found valid LitellmUserRoles '{role.value}' from SSO attribute '{generic_user_role_attribute_name}'" + f"Determined role '{user_role.value if user_role else None}' from groups '{user_groups}' using role_mappings" ) + else: + # No groups found, use default_role + user_role = role_mappings.default_role + verbose_proxy_logger.debug( + f"No groups found in '{group_claim}', using default_role: {role_mappings.default_role}" + ) + + # Fallback to existing logic if role_mappings not used + if user_role is None: + user_role_from_sso = get_nested_value(response, generic_user_role_attribute_name) + if user_role_from_sso is not None: + role = get_litellm_user_role(user_role_from_sso) + if role is not None: + user_role = role + verbose_proxy_logger.debug( + f"Found valid LitellmUserRoles '{role.value}' from SSO attribute '{generic_user_role_attribute_name}'" + ) return CustomOpenID( id=get_nested_value(response, generic_user_id_attribute_name), @@ -369,6 +454,40 @@ async def get_generic_sso_response( userinfo_endpoint=generic_userinfo_endpoint, ) + # Get role_mappings from SSO settings if available + role_mappings: Optional["RoleMappings"] = None + try: + from litellm.proxy.utils import get_prisma_client_or_throw + + prisma_client = get_prisma_client_or_throw( + "Prisma client is None, connect a database to your proxy" + ) + + # Get SSO config from dedicated table + sso_db_record = await prisma_client.db.litellm_ssoconfig.find_unique( + where={"id": "sso_config"} + ) + + if sso_db_record and sso_db_record.sso_settings: + sso_settings_dict = dict(sso_db_record.sso_settings) + role_mappings_data = sso_settings_dict.get("role_mappings") + + if role_mappings_data: + from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings + if isinstance(role_mappings_data, dict): + role_mappings = RoleMappings(**role_mappings_data) + elif isinstance(role_mappings_data, RoleMappings): + role_mappings = role_mappings_data + + verbose_proxy_logger.debug( + f"Loaded role_mappings for provider '{role_mappings.provider}'" + ) + except Exception as e: + # If we can't load role_mappings, continue with existing logic + verbose_proxy_logger.debug( + f"Could not load role_mappings from database: {e}. Continuing with existing role logic." + ) + def response_convertor(response, client): nonlocal received_response # return for user debugging received_response = response @@ -376,6 +495,7 @@ async def get_generic_sso_response( response=response, jwt_handler=jwt_handler, sso_jwt_handler=sso_jwt_handler, + role_mappings=role_mappings, ) SSOProvider = create_provider( From bf76e66d2ccaace63eb6353ab92e37b57d99b349 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 09:42:59 -0800 Subject: [PATCH 005/131] Working SSO Mapping new user, overrides default user settings --- litellm/proxy/management_endpoints/ui_sso.py | 48 +++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 7a404bcda81..5afccc6fe5f 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -1173,8 +1173,44 @@ async def insert_sso_user( if user_defined_values is None: raise ValueError("user_defined_values is None") + # Check if role_mappings is configured in SSO settings + role_mappings_configured = False + try: + from litellm.proxy.utils import get_prisma_client_or_throw + + prisma_client = get_prisma_client_or_throw( + "Prisma client is None, connect a database to your proxy" + ) + + # Get SSO config from dedicated table + sso_db_record = await prisma_client.db.litellm_ssoconfig.find_unique( + where={"id": "sso_config"} + ) + + if sso_db_record and sso_db_record.sso_settings: + sso_settings_dict = dict(sso_db_record.sso_settings) + role_mappings_data = sso_settings_dict.get("role_mappings") + role_mappings_configured = role_mappings_data is not None + except Exception as e: + # If we can't check role_mappings, continue with existing logic + verbose_proxy_logger.debug( + f"Could not check role_mappings configuration: {e}. Using default behavior." + ) + + # Apply default_internal_user_params if litellm.default_internal_user_params: - user_defined_values.update(litellm.default_internal_user_params) # type: ignore + # If role_mappings is configured and user_role is already set from SSO, preserve it + if role_mappings_configured and user_defined_values.get("user_role") is not None: + # Preserve the SSO-extracted role, but apply other defaults + preserved_role = user_defined_values.get("user_role") + user_defined_values.update(litellm.default_internal_user_params) # type: ignore + user_defined_values["user_role"] = preserved_role # Restore preserved role + verbose_proxy_logger.debug( + f"Preserved SSO-extracted role '{preserved_role}' (role_mappings configured)" + ) + else: + # Default behavior: update all values including role + user_defined_values.update(litellm.default_internal_user_params) # type: ignore # Set budget for internal users if user_defined_values.get("user_role") == LitellmUserRoles.INTERNAL_USER.value: @@ -1812,7 +1848,15 @@ class SSOAuthenticationHandler: ) user_id = getattr(result, "id", None) user_email = getattr(result, "email", None) - user_role = getattr(result, generic_user_role_attribute_name, None) # type: ignore + if user_role is None: + _role_from_attr = getattr(result, generic_user_role_attribute_name, None) # type: ignore + if _role_from_attr is not None: + # Convert enum to string if needed + user_role = ( + _role_from_attr.value + if isinstance(_role_from_attr, LitellmUserRoles) + else _role_from_attr + ) if user_id is None and result is not None: _first_name = getattr(result, "first_name", "") or "" From 313a613a13e2756308ce828924864815ac36bd38 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 10:56:11 -0800 Subject: [PATCH 006/131] Adding tests --- .../proxy/management_endpoints/test_ui_sso.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 500fc67de89..20829466570 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -3043,3 +3043,108 @@ class TestAddMissingTeamMember: assert set(added_teams) == set( expected_teams_added ), f"Expected teams {expected_teams_added}, but got {added_teams}" + + +@pytest.mark.asyncio +async def test_role_mappings_override_default_internal_user_params(): + """ + Test that when role_mappings is configured in SSO settings, + the SSO-extracted role overrides default_internal_user_params role. + """ + from litellm.proxy._types import NewUserResponse, SSOUserDefinedValues + from litellm.proxy.management_endpoints.ui_sso import insert_sso_user + + # Save original default_internal_user_params + original_default_params = getattr(litellm, "default_internal_user_params", None) + + try: + # Set default_internal_user_params with a role that should be overridden + litellm.default_internal_user_params = { + "user_role": "internal_user", + "max_budget": 100, + "budget_duration": "30d", + "models": ["gpt-3.5-turbo"], + } + + # Mock SSO result + mock_result_openid = CustomOpenID( + id="test-user-123", + email="test@example.com", + display_name="Test User", + provider="microsoft", + team_ids=[], + ) + + # User defined values with SSO-extracted role (from role_mappings) + user_defined_values: SSOUserDefinedValues = { + "user_id": "test-user-123", + "user_email": "test@example.com", + "user_role": "proxy_admin", # Role from SSO role_mappings + "max_budget": None, + "budget_duration": None, + "models": [], + } + + # Mock Prisma client with SSO config that has role_mappings configured + mock_prisma = MagicMock() + mock_sso_config = MagicMock() + mock_sso_config.sso_settings = { + "role_mappings": { + "Admin": "proxy_admin", + "User": "internal_user", + } + } + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock( + return_value=mock_sso_config + ) + + # Mock new_user function + mock_new_user_response = NewUserResponse( + user_id="test-user-123", + key="sk-xxxxx", + teams=None, + ) + + with patch( + "litellm.proxy.utils.get_prisma_client_or_throw", + return_value=mock_prisma, + ), patch( + "litellm.proxy.management_endpoints.ui_sso.new_user", + return_value=mock_new_user_response, + ) as mock_new_user: + # Act + result = await insert_sso_user( + result_openid=mock_result_openid, + user_defined_values=user_defined_values, + ) + + # Assert - verify new_user was called with preserved SSO role + mock_new_user.assert_called_once() + call_args = mock_new_user.call_args + new_user_request = call_args.kwargs["data"] + + # The role from SSO should be preserved, not overridden by default_internal_user_params + assert ( + new_user_request.user_role == "proxy_admin" + ), "SSO-extracted role should override default_internal_user_params role" + + # Other default params should still be applied + assert ( + new_user_request.max_budget == 100 + ), "max_budget from default_internal_user_params should be applied" + assert ( + new_user_request.budget_duration == "30d" + ), "budget_duration from default_internal_user_params should be applied" + + # Note: models are applied via _update_internal_new_user_params inside new_user, + # not in insert_sso_user, so we verify user_defined_values was updated correctly + # by checking that the function completed successfully and other defaults were applied + # The models will be applied when new_user processes the request + + finally: + # Restore original default_internal_user_params + if original_default_params is not None: + litellm.default_internal_user_params = original_default_params + else: + if hasattr(litellm, "default_internal_user_params"): + delattr(litellm, "default_internal_user_params") From e92db2d718a78dc0443c801cb8467e6a5160c725 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 13:04:42 -0800 Subject: [PATCH 007/131] Improve e2e flow for cloudzero --- .../hooks/cloudzero/useCloudZeroSettings.ts | 70 +++++++++++++++---- .../CloudZeroCostTracking.tsx | 4 +- .../CloudZeroEmptyPlaceholder.test.tsx | 2 +- .../CloudZeroEmptyPlaceholder.tsx | 2 +- .../CloudZeroIntegrationSettings.tsx | 8 ++- .../components/CloudZeroCostTracking/types.ts | 8 +-- 6 files changed, 71 insertions(+), 23 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts index 5ccbe244e60..96f5ab2f944 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts @@ -17,19 +17,35 @@ const getCloudZeroSettings = async (accessToken: string): Promise ({})); - const errorMessage = - errorData?.error?.message || errorData?.message || errorData?.detail || "Failed to fetch CloudZero settings"; + let errorMessage = "Failed to fetch CloudZero settings"; + try { + const errorData = await response.json(); + // Handle different error response formats + if (typeof errorData === "object" && errorData !== null) { + errorMessage = + errorData?.error?.message || + errorData?.error || + errorData?.message || + errorData?.detail || + (typeof errorData?.error === "string" ? errorData.error : errorMessage); + } else if (typeof errorData === "string") { + errorMessage = errorData; + } + } catch { + // If JSON parsing fails, use the status text + errorMessage = response.statusText || errorMessage; + } throw new Error(errorMessage); } const data = await response.json(); + + // Check if settings are actually configured (all required fields are present) + if (!data || (!data.api_key_masked && !data.connection_id)) { + return null; + } + return data; }; @@ -77,9 +93,22 @@ const updateCloudZeroSettings = async (accessToken: string, params: UpdateParams }); if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - const errorMessage = - errorData?.error?.message || errorData?.message || errorData?.detail || "Failed to update CloudZero settings"; + let errorMessage = "Failed to update CloudZero settings"; + try { + const errorData = await response.json(); + if (typeof errorData === "object" && errorData !== null) { + errorMessage = + errorData?.error?.message || + errorData?.error || + errorData?.message || + errorData?.detail || + (typeof errorData?.error === "string" ? errorData.error : errorMessage); + } else if (typeof errorData === "string") { + errorMessage = errorData; + } + } catch { + errorMessage = response.statusText || errorMessage; + } throw new Error(errorMessage); } @@ -117,9 +146,22 @@ const deleteCloudZeroSettings = async (accessToken: string): Promise ({})); - const errorMessage = - errorData?.error?.message || errorData?.message || errorData?.detail || "Failed to delete CloudZero settings"; + let errorMessage = "Failed to delete CloudZero settings"; + try { + const errorData = await response.json(); + if (typeof errorData === "object" && errorData !== null) { + errorMessage = + errorData?.error?.message || + errorData?.error || + errorData?.message || + errorData?.detail || + (typeof errorData?.error === "string" ? errorData.error : errorMessage); + } else if (typeof errorData === "string") { + errorMessage = errorData; + } + } catch { + errorMessage = response.statusText || errorMessage; + } throw new Error(errorMessage); } diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx index fbb892cb1d8..db3ea94bbf9 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx @@ -36,7 +36,9 @@ export default function CloudZeroCostTracking() { if (error) { return ( - Error loading CloudZero settings: {error.message} + + Error loading CloudZero settings: {error instanceof Error ? error.message : String(error)} + ); } diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx index 04e0a67dea6..f7b90884006 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx @@ -9,6 +9,6 @@ describe("CloudZeroEmptyPlaceholder", () => { expect(screen.getByText("No CloudZero Integration Found")).toBeInTheDocument(); expect(screen.getByText(/Connect your CloudZero account/)).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Create Integration" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Add CloudZero Integration" })).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx index 1719a949b86..aca074dc290 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx @@ -21,7 +21,7 @@ export default function CloudZeroEmptyPlaceholder({ startCreation }: CloudZeroEm } > diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx index 780fa83652a..c161d241f7d 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx @@ -134,10 +134,14 @@ export function CloudZeroIntegrationSettings({ settings, onSettingsUpdated }: Cl }} > - {settings.api_key_masked} + + {settings.api_key_masked || Not configured} + - {settings.connection_id} + + {settings.connection_id || Not configured} + {settings.timezone || Default (UTC)} diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts index a41afee4f72..ed3c76cc3b1 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts @@ -1,6 +1,6 @@ export interface CloudZeroSettings { - api_key_masked: string; - connection_id: string; - timezone?: string; - status?: string; + api_key_masked: string | null; + connection_id: string | null; + timezone?: string | null; + status?: string | null; } From 74842de78eddfdea22448b85aec9a0974b28a36f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 13:04:59 -0800 Subject: [PATCH 008/131] Adding backend --- .../spend_tracking/cloudzero_endpoints.py | 34 ++++-- litellm/types/proxy/cloudzero_endpoints.py | 8 +- .../test_cloudzero_endpoints.py | 111 +++++++++++++++++- 3 files changed, 137 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/spend_tracking/cloudzero_endpoints.py b/litellm/proxy/spend_tracking/cloudzero_endpoints.py index 2cf4ce8f16a..172169f2c7a 100644 --- a/litellm/proxy/spend_tracking/cloudzero_endpoints.py +++ b/litellm/proxy/spend_tracking/cloudzero_endpoints.py @@ -69,7 +69,7 @@ async def _get_cloudzero_settings(): Retrieve CloudZero settings from the database with decrypted API key. Returns: - dict: CloudZero settings with decrypted API key + dict: CloudZero settings with decrypted API key, or empty dict if not configured """ from litellm.proxy.proxy_server import prisma_client @@ -82,10 +82,16 @@ async def _get_cloudzero_settings(): cloudzero_config = await prisma_client.db.litellm_config.find_first( where={"param_name": "cloudzero_settings"} ) - if cloudzero_config is None: + if cloudzero_config is None or cloudzero_config.param_value is None: return {} - settings = dict(cloudzero_config.param_value) + # Handle both dict and JSON string cases + if isinstance(cloudzero_config.param_value, dict): + settings = cloudzero_config.param_value + elif isinstance(cloudzero_config.param_value, str): + settings = json.loads(cloudzero_config.param_value) + else: + settings = dict(cloudzero_config.param_value) # Decrypt the API key encrypted_api_key = settings.get("api_key") @@ -119,6 +125,7 @@ async def get_cloudzero_settings( Returns the current CloudZero configuration with the API key masked for security. Only the first 4 and last 4 characters of the API key are shown. + Returns null/empty values when settings are not configured (consistent with other settings endpoints). Only admin users can view CloudZero settings. """ @@ -133,22 +140,27 @@ async def get_cloudzero_settings( # Get CloudZero settings using the accessor method settings = await _get_cloudzero_settings() + # If settings are empty, return null/empty values (consistent with other endpoints) + if not settings: + return CloudZeroSettingsView( + api_key_masked=None, + connection_id=None, + timezone=None, + status=None, + ) + # Use SensitiveDataMasker to mask the API key masked_settings = _sensitive_masker.mask_dict(settings) return CloudZeroSettingsView( - api_key_masked=masked_settings["api_key"], - connection_id=settings["connection_id"], - timezone=settings["timezone"], + api_key_masked=masked_settings.get("api_key"), + connection_id=settings.get("connection_id"), + timezone=settings.get("timezone"), status="configured", ) except HTTPException as e: - if e.status_code == 400: - # Settings not configured - raise HTTPException( - status_code=404, detail={"error": "CloudZero settings not configured"} - ) + # Re-raise HTTPExceptions as-is raise e except Exception as e: verbose_proxy_logger.error(f"Error retrieving CloudZero settings: {str(e)}") diff --git a/litellm/types/proxy/cloudzero_endpoints.py b/litellm/types/proxy/cloudzero_endpoints.py index 1d909bf7f8c..fc48717e80a 100644 --- a/litellm/types/proxy/cloudzero_endpoints.py +++ b/litellm/types/proxy/cloudzero_endpoints.py @@ -45,10 +45,10 @@ class CloudZeroExportResponse(BaseModel): class CloudZeroSettingsView(BaseModel): """Response model for viewing CloudZero settings with masked API key""" - api_key_masked: str = Field(..., description="Masked API key showing only first 4 and last 4 characters") - connection_id: str = Field(..., description="CloudZero connection ID for data submission") - timezone: str = Field(..., description="Timezone for date handling") - status: str = Field(..., description="Configuration status") + api_key_masked: Optional[str] = Field(None, description="Masked API key showing only first 4 and last 4 characters") + connection_id: Optional[str] = Field(None, description="CloudZero connection ID for data submission") + timezone: Optional[str] = Field(None, description="Timezone for date handling") + status: Optional[str] = Field(None, description="Configuration status") class CloudZeroSettingsUpdate(BaseModel): diff --git a/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py index 8ff5774bf50..6d460f63332 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py @@ -1,6 +1,6 @@ import os import sys -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient @@ -77,3 +77,112 @@ async def test_delete_cloudzero_settings_not_found(client, monkeypatch): finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + +@pytest.mark.asyncio +async def test_get_cloudzero_settings_success(client, monkeypatch): + """Test GET /cloudzero/settings returns settings when configured""" + mock_config = MagicMock() + mock_config.param_name = "cloudzero_settings" + mock_config.param_value = { + "api_key": "encrypted_key", + "connection_id": "conn_123", + "timezone": "UTC" + } + + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=mock_config) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + # Mock the decrypt function to return a decrypted key + with patch("litellm.proxy.spend_tracking.cloudzero_endpoints.decrypt_value_helper") as mock_decrypt: + mock_decrypt.return_value = "decrypted_api_key" + + # Mock the masker + with patch("litellm.proxy.spend_tracking.cloudzero_endpoints._sensitive_masker") as mock_masker: + mock_masker.mask_dict.return_value = {"api_key": "test****key"} + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.get("/cloudzero/settings") + assert response.status_code == 200 + data = response.json() + assert data["connection_id"] == "conn_123" + assert data["timezone"] == "UTC" + assert data["status"] == "configured" + assert data["api_key_masked"] == "test****key" + mock_litellm_config.find_first.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_get_cloudzero_settings_not_configured(client, monkeypatch): + """Test GET /cloudzero/settings returns 200 with null values when not configured (consistent with other endpoints)""" + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=None) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.get("/cloudzero/settings") + # Should return 200 with null values (not 404) - consistent with other settings endpoints + assert response.status_code == 200 + data = response.json() + assert data["api_key_masked"] is None + assert data["connection_id"] is None + assert data["timezone"] is None + assert data["status"] is None + mock_litellm_config.find_first.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_get_cloudzero_settings_empty_param_value(client, monkeypatch): + """Test GET /cloudzero/settings returns 200 with null values when param_value is None""" + mock_config = MagicMock() + mock_config.param_name = "cloudzero_settings" + mock_config.param_value = None + + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=mock_config) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.get("/cloudzero/settings") + # Should return 200 with null values (not 404) - consistent with other settings endpoints + assert response.status_code == 200 + data = response.json() + assert data["api_key_masked"] is None + assert data["connection_id"] is None + assert data["timezone"] is None + assert data["status"] is None + mock_litellm_config.find_first.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + From ffcac2eebcc3a1f2c74a81171d79c930750f0ee8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 18:04:04 -0800 Subject: [PATCH 009/131] Allow deleting key expiry --- .../key_management_endpoints.py | 13 ++++++-- tests/proxy_unit_tests/test_proxy_utils.py | 5 +++ .../test_key_management_endpoints.py | 31 +++++++++++++++++++ .../KeyLifecycleSettings.tsx | 4 +-- 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 14d221d19e1..cc2ac908149 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -507,7 +507,11 @@ async def _common_key_generation_helper( # noqa: PLR0915 upperbound_duration = duration_in_seconds( duration=upperbound_value ) - user_duration = duration_in_seconds(duration=value) + # Handle special case where duration is "-1" (never expires) + if value == "-1": + user_duration = float('inf') # Infinite duration + else: + user_duration = duration_in_seconds(duration=value) if user_duration > upperbound_duration: raise HTTPException( status_code=400, @@ -1339,7 +1343,10 @@ async def prepare_key_update_data( if "duration" in non_default_values: duration = non_default_values.pop("duration") - if duration and (isinstance(duration, str)) and len(duration) > 0: + if duration == "-1": + # Set expires to None to indicate the key never expires + non_default_values["expires"] = None + elif duration and (isinstance(duration, str)) and len(duration) > 0: duration_s = duration_in_seconds(duration=duration) expires = datetime.now(timezone.utc) + timedelta(seconds=duration_s) non_default_values["expires"] = expires @@ -1452,7 +1459,7 @@ async def update_key_fn( - tpm_limit_type: Optional[str] - TPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic" - rpm_limit_type: Optional[str] - RPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic" - allowed_cache_controls: Optional[list] - List of allowed cache control values - - duration: Optional[str] - Key validity duration ("30d", "1h", etc.) + - duration: Optional[str] - Key validity duration ("30d", "1h", etc.) or "-1" to never expire - permissions: Optional[dict] - Key-specific permissions - send_invite_email: Optional[bool] - Send invite email to user_id - guardrails: Optional[List[str]] - List of active guardrails for the key diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 2e5cfff8bf0..5c3c3948920 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -678,6 +678,11 @@ async def test_prepare_key_update_data(): updated_data = await prepare_key_update_data(data, existing_key_row) assert updated_data["metadata"] is None + # Test duration "-1" sets expires to None (never expires) + data = UpdateKeyRequest(key="test_key", duration="-1") + updated_data = await prepare_key_update_data(data, existing_key_row) + assert updated_data["expires"] is None + @pytest.mark.parametrize( "env_vars, expected_url", diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index ff85e6d9e73..648045a7ea6 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -815,6 +815,37 @@ async def test_update_service_account_works_with_team_id(): await prepare_key_update_data(data=data, existing_key_row=existing_key) +@pytest.mark.asyncio +async def test_prepare_key_update_data_duration_never_expires(): + """Test that duration="-1" sets expires to None (never expires).""" + from litellm.proxy._types import UpdateKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import ( + prepare_key_update_data, + ) + + # Mock existing key + existing_key = LiteLLM_VerificationToken( + token="test-token", + key_alias="test-key", + models=["gpt-3.5-turbo"], + user_id="test-user", + team_id=None, + auto_rotate=False, + rotation_interval=None, + metadata={}, + ) + + # Test setting duration to "-1" (never expires) + update_request = UpdateKeyRequest(key="test-token", duration="-1") + + result = await prepare_key_update_data( + data=update_request, existing_key_row=existing_key + ) + + # Verify that expires is set to None + assert result["expires"] is None + + @pytest.mark.asyncio async def test_validate_team_id_used_in_service_account_request_requires_team_id(): """ diff --git a/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.tsx b/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.tsx index 8129f4314fc..81d22b56347 100644 --- a/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.tsx +++ b/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.tsx @@ -64,13 +64,13 @@ const KeyLifecycleSettings: React.FC = ({
Date: Fri, 19 Dec 2025 23:13:36 -0800 Subject: [PATCH 010/131] fix(gemini): properly catch context window exceeded errors Fixes #18282 This PR fixes two issues with Gemini context window error handling: 1. **Pattern matching for Gemini 2.0 Flash**: The previous pattern 'input token count exceeds the maximum number of tokens allowed' doesn't match Gemini 2.0 Flash errors which include dynamic token counts like '(2800010)' in the message. Split into shorter patterns that work with both formats. 2. **Add context window check to Gemini block**: The is_error_str_context_window_exceeded() check was only called for OpenAI-compatible providers, not for Gemini/Vertex AI. Added the check to the Gemini-specific error handling block. Test cases added for both Gemini 2.0 Flash and 2.5/3 error formats. --- .../exception_mapping_utils.py | 12 +++- .../test_exception_mapping_utils.py | 60 ++++++++++++++++++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 7bf95ca3404..1517d1e776d 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -78,9 +78,7 @@ class ExceptionCheckers: "is longer than the model's context length", "input tokens exceed the configured limit", "`inputs` tokens + `max_new_tokens` must be", - # Gemini pattern: "The input token count exceeds the maximum number of tokens allowed" - # See: https://github.com/BerriAI/litellm/issues/XXXX - "input token count exceeds the maximum number of tokens allowed", + "exceeds the maximum number of tokens allowed", # Gemini ] for substring in known_exception_substrings: if substring in _error_str_lowercase: @@ -1262,6 +1260,14 @@ def exception_type( # type: ignore # noqa: PLR0915 model=model, llm_provider=custom_llm_provider, ) + elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"ContextWindowExceededError: {custom_llm_provider.capitalize()}Exception - {error_str}", + model=model, + llm_provider=custom_llm_provider, + litellm_debug_info=extra_information, + ) elif ( "None Unknown Error." in error_str or "Content has no parts." in error_str diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index f69b9c35236..9e742a83c6a 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -40,8 +40,7 @@ context_window_test_cases = [ "`inputs` tokens + `max_new_tokens` must be <= 4096", True, ), - # Gemini context window error format - # See: https://github.com/BerriAI/litellm/issues/XXXX + # Gemini 2.5/3 format ( "The input token count exceeds the maximum number of tokens allowed 1048576.", True, @@ -50,6 +49,15 @@ context_window_test_cases = [ "GeminiException BadRequestError - {\n \"error\": {\n \"code\": 400,\n \"message\": \"The input token count exceeds the maximum number of tokens allowed 1048576.\",\n \"status\": \"INVALID_ARGUMENT\"\n }\n}\n", True, ), + # Gemini 2.0 Flash format (includes input token count in message) + ( + "The input token count (2800010) exceeds the maximum number of tokens allowed (1048575).", + True, + ), + ( + "GeminiException BadRequestError - {\n \"error\": {\n \"code\": 400,\n \"message\": \"The input token count (2800010) exceeds the maximum number of tokens allowed (1048575).\",\n \"status\": \"INVALID_ARGUMENT\"\n }\n}\n", + True, + ), # Test case insensitivity ("ERROR: THIS MODEL'S MAXIMUM CONTEXT LENGTH IS 1024.", True), # Cerebras context window error format @@ -169,6 +177,54 @@ class TestExceptionCheckers: result = ExceptionCheckers.is_azure_content_policy_violation_error(error_str) assert result is False, f"Should NOT detect policy violation in: {error_str}" +gemini_context_window_test_cases = [ + # Gemini 2.0 Flash format (includes input token count in message) + ( + "The input token count (2800010) exceeds the maximum number of tokens allowed (1048575).", + True, + ), + # Gemini 2.5/3 format + ( + "The input token count exceeds the maximum number of tokens allowed (1048576).", + True, + ), + ("A generic error occurred.", False), +] + + +@pytest.mark.parametrize( + "error_message, should_raise_context_window", gemini_context_window_test_cases +) +def test_gemini_context_window_error_mapping(error_message, should_raise_context_window): + """ + Tests that the exception_type function correctly maps Gemini's + context window exceeded errors to litellm.ContextWindowExceededError. + """ + model = "gemini/gemini-2.0-flash" + custom_llm_provider = "gemini" + + # Create a generic exception with the specific error message + original_exception = Exception(error_message) + + if should_raise_context_window: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model=model, + original_exception=original_exception, + custom_llm_provider=custom_llm_provider, + ) + # Check if the raised exception is indeed a ContextWindowExceededError + assert isinstance(excinfo.value, litellm.ContextWindowExceededError) + else: + # For the negative case, we expect it to raise a generic APIConnectionError + with pytest.raises(litellm.APIConnectionError): + exception_type( + model=model, + original_exception=original_exception, + custom_llm_provider=custom_llm_provider, + ) + + # Test cases for Vertex AI RateLimitError mapping # As per https://github.com/BerriAI/litellm/issues/16189 vertex_rate_limit_test_cases = [ From 9002f75277228c0723d0d503f71f1f1a5de4638f Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 20 Dec 2025 11:01:41 -0600 Subject: [PATCH 011/131] Require auth for MCP connection test --- .../mcp_server/rest_endpoints.py | 8 +-- .../mcp_server/test_rest_endpoints.py | 55 +++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 032331ece02..92c390f0a64 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1,5 +1,4 @@ import importlib -import traceback from typing import Dict, List, Optional, Union from fastapi import APIRouter, Depends, Query, Request @@ -329,16 +328,15 @@ if MCP_AVAILABLE: except Exception as e: verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True) - stack_trace = traceback.format_exc() return { "status": "error", - "message": f"An internal error has occurred: {str(e)}", - "stack_trace": stack_trace, + "message": "An internal error has occurred while testing the MCP server.", } - @router.post("/test/connection") + @router.post("/test/connection", dependencies=[Depends(user_api_key_auth)]) async def test_connection( request: NewMCPServerRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ Test if we can connect to the provided MCP server before adding it diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index a0c09663a88..85ec807b1ff 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -7,6 +7,7 @@ from litellm.proxy._experimental.mcp_server import rest_endpoints from litellm.proxy._experimental.mcp_server.auth import ( user_api_key_auth_mcp as auth_mcp, ) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy._types import NewMCPServerRequest, UserAPIKeyAuth from litellm.types.mcp import MCPAuth @@ -31,6 +32,60 @@ def _build_request(headers: Optional[Dict[str, str]] = None) -> Request: return Request(scope, receive=receive) +def _get_route(path: str, method: str): + for route in rest_endpoints.router.routes: + if getattr(route, "path", None) == path and method in getattr( + route, "methods", set() + ): + return route + raise AssertionError(f"Route {method} {path} not found") + + +def _route_has_dependency(route, dependency) -> bool: + if any( + getattr(dep, "dependency", None) == dependency + for dep in getattr(route, "dependencies", []) + ): + return True + dependant = getattr(route, "dependant", None) + if dependant is None: + return False + return any(getattr(dep, "call", None) == dependency for dep in dependant.dependencies) + + +@pytest.mark.asyncio +async def test_execute_with_mcp_client_redacts_stack_trace(monkeypatch): + def fake_create_client(*args, **kwargs): + return object() + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_create_mcp_client", + fake_create_client, + ) + + async def failing_operation(client): + raise RuntimeError("boom") + + payload = NewMCPServerRequest( + server_name="example", + url="https://example.com", + auth_type=MCPAuth.none, + ) + + result = await rest_endpoints._execute_with_mcp_client( + payload, failing_operation + ) + + assert result["status"] == "error" + assert "stack_trace" not in result + + +def test_test_connection_requires_auth_dependency(): + route = _get_route("/mcp-rest/test/connection", "POST") + assert _route_has_dependency(route, user_api_key_auth) + + @pytest.mark.asyncio async def test_test_tools_list_forwards_mcp_auth_header(monkeypatch): """Ensure credential-based auth forwards the auth_value to the MCP client.""" From acce6b9c83f143116038b49be710398802cd540e Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 20 Dec 2025 15:56:46 -0600 Subject: [PATCH 012/131] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../proxy/_experimental/mcp_server/test_rest_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 85ec807b1ff..31ab4afb631 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -82,7 +82,7 @@ async def test_execute_with_mcp_client_redacts_stack_trace(monkeypatch): def test_test_connection_requires_auth_dependency(): - route = _get_route("/mcp-rest/test/connection", "POST") + route = _get_route("/test/connection", "POST") assert _route_has_dependency(route, user_api_key_auth) From 901d145b1a839be53c5149f80f263bd466576d0c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:37:13 -0800 Subject: [PATCH 013/131] Adding UI portion for Agents MD --- AGENTS.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 2c778dc0d71..61afbd035fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,27 @@ LiteLLM is a unified interface for 100+ LLMs that: - Test provider-specific functionality thoroughly - Consider adding load tests for performance-critical changes +### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND) + +1. **Use Common Components as much as possible**: + - These are usually defined in the `common_components` directory + - Use these components as much as possible and avoid building new components unless needed + - Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible + +2. **Testing**: + - The codebase uses **Vitest** and **React Testing Library** + - **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId` + - **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`) + - **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled + - **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present + - **Test names must start with "should"**: All test names should follow the pattern `it("should ...")` + - **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed + - **Structure tests properly**: + - First test should verify the component renders successfully + - Subsequent tests should focus on functionality and user interactions + - Use `waitFor` for async operations that aren't already awaited + - **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation + ### IMPORTANT PATTERNS 1. **Function/Tool Calling**: From f747d12a5f8f9096a4c1cd1e3bd96dee45ff7802 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:51:31 -0800 Subject: [PATCH 014/131] minor styling changes --- .../src/components/cache_dashboard.tsx | 22 +++-- .../organization/organization_view.test.tsx | 23 ++++- .../organization/organization_view.tsx | 87 ++++++++++--------- .../components/team/member_permissions.tsx | 4 +- .../src/components/team/team_info.tsx | 2 +- 5 files changed, 82 insertions(+), 56 deletions(-) diff --git a/ui/litellm-dashboard/src/components/cache_dashboard.tsx b/ui/litellm-dashboard/src/components/cache_dashboard.tsx index 38c0f1a8f41..7b57191a879 100644 --- a/ui/litellm-dashboard/src/components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/cache_dashboard.tsx @@ -1,23 +1,23 @@ -import React, { useState, useEffect } from "react"; import { - Card, BarChart, - Subtitle, - Grid, + Card, Col, DateRangePickerValue, + Grid, + Icon, MultiSelect, MultiSelectItem, - TabPanel, - TabPanels, + Subtitle, + Tab, TabGroup, TabList, - Tab, - Icon, + TabPanel, + TabPanels, Text, } from "@tremor/react"; -import UsageDatePicker from "./shared/usage_date_picker"; +import React, { useEffect, useState } from "react"; import NotificationsManager from "./molecules/notifications_manager"; +import UsageDatePicker from "./shared/usage_date_picker"; import { RefreshIcon } from "@heroicons/react/outline"; import { adminGlobalCacheActivity, cachingHealthCheckCall } from "./networking"; @@ -271,9 +271,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole
Cache Analytics - - 
Cache Health
- + Cache Health Cache Settings
diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx index 0be03169e89..5204efc9411 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { render, waitFor } from "@testing-library/react"; +import { render, screen, waitFor } from "@testing-library/react"; import { vi, test, expect } from "vitest"; import OrganizationInfoView from "./organization_view"; @@ -82,3 +82,24 @@ test("renders organization view after loading data", async () => { expect(findAllByText("Acme Corp")).toBeTruthy(); }); }); + +test("should display empty state when organization has no members", async () => { + const { organizationInfoCall } = await import("../networking"); + (organizationInfoCall as unknown as ReturnType).mockResolvedValueOnce(mockOrg); + + render( + {}} + accessToken="test-token" + is_org_admin={false} + is_proxy_admin={false} + userModels={[]} + editOrg={false} + />, + ); + + await waitFor(() => { + expect(screen.getByText("No members found")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.tsx index 962ec6fa4ea..595987aadf5 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.tsx @@ -324,7 +324,6 @@ const OrganizationInfoView: React.FC = ({ - {/* Budget Panel */}
@@ -340,47 +339,55 @@ const OrganizationInfoView: React.FC = ({ - {orgData.members?.map((member, index) => ( - - - {member.user_id} - - - {member.user_role} - - - ${formatNumberWithCommas(member.spend, 4)} - - - {new Date(member.created_at).toLocaleString()} - - - {canEditOrg && ( - <> - { - setSelectedEditMember({ - role: member.user_role, - user_email: member.user_email, - user_id: member.user_id, - }); - setIsEditMemberModalVisible(true); - }} - /> - { - handleMemberDelete(member); - }} - /> - - )} + {orgData.members && orgData.members.length > 0 ? ( + orgData.members.map((member, index) => ( + + + {member.user_id} + + + {member.user_role} + + + ${formatNumberWithCommas(member.spend, 4)} + + + {new Date(member.created_at).toLocaleString()} + + + {canEditOrg && ( + <> + { + setSelectedEditMember({ + role: member.user_role, + user_email: member.user_email, + user_id: member.user_id, + }); + setIsEditMemberModalVisible(true); + }} + /> + { + handleMemberDelete(member); + }} + /> + + )} + + + )) + ) : ( + + + No members found - ))} + )} diff --git a/ui/litellm-dashboard/src/components/team/member_permissions.tsx b/ui/litellm-dashboard/src/components/team/member_permissions.tsx index 6a7ab541ddf..7eefedb4a2f 100644 --- a/ui/litellm-dashboard/src/components/team/member_permissions.tsx +++ b/ui/litellm-dashboard/src/components/team/member_permissions.tsx @@ -94,9 +94,9 @@ const MemberPermissions: React.FC = ({ teamId, accessTok - +
)}
diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index 49a04cce1d1..d2d1c885931 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -508,7 +508,7 @@ const TeamInfoView: React.FC = ({ Back to Teams {info.team_alias} -
+
{info.team_id}
diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx index a46738ab365..83393c4a94b 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx @@ -1,11 +1,11 @@ import { isAdminRole } from "@/utils/roles"; import { QuestionCircleOutlined } from "@ant-design/icons"; -import { useQuery } from "@tanstack/react-query"; import { Button, Tab, TabGroup, TabList, TabPanel, TabPanels, Text, Title } from "@tremor/react"; import { Descriptions, Modal, Select, Tooltip, Typography } from "antd"; import React, { useEffect, useState } from "react"; +import { useMCPServers } from "../../app/(dashboard)/hooks/mcpServers/useMCPServers"; import NotificationsManager from "../molecules/notifications_manager"; -import { deleteMCPServer, fetchMCPServers } from "../networking"; +import { deleteMCPServer } from "../networking"; import { DataTable } from "../view_logs/table"; import CreateMCPServer from "./create_mcp_server"; import MCPConnect from "./mcp_connect"; @@ -19,19 +19,7 @@ const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state"; const { Option } = Select; const MCPServers: React.FC = ({ accessToken, userRole, userID }) => { - const { - data: mcpServers, - isLoading: isLoadingServers, - refetch, - dataUpdatedAt, - } = useQuery({ - queryKey: ["mcpServers"], - queryFn: () => { - if (!accessToken) throw new Error("Access Token required"); - return fetchMCPServers(accessToken); - }, - enabled: !!accessToken, - }) as { data: MCPServer[]; isLoading: boolean; refetch: () => void; dataUpdatedAt: number }; + const { data: mcpServers, isLoading: isLoadingServers, refetch, dataUpdatedAt } = useMCPServers(accessToken); // Log allowed_tools from fetched servers React.useEffect(() => { diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 3c0a0f520d4..f3ad803e27c 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1,43 +1,40 @@ "use client"; -import React, { useState, useEffect, useCallback } from "react"; -import { Button, TextInput, Grid, Col } from "@tremor/react"; -import { Text, Title, Accordion, AccordionHeader, AccordionBody } from "@tremor/react"; -import { CopyToClipboard } from "react-copy-to-clipboard"; -import { Button as Button2, Modal, Form, Input, Select, Radio, Switch } from "antd"; -import NumericalInput from "../shared/numerical_input"; -import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; -import SchemaFormFields from "../common_components/check_openapi_schema"; -import { - keyCreateCall, - modelAvailableCall, - getGuardrailsList, - proxyBaseUrl, - getPossibleUserRoles, - userFilterUICall, - keyCreateServiceAccountCall, - fetchMCPAccessGroups, - getPromptsList, -} from "../networking"; -import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; -import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; -import { Team } from "../key_team_helpers/key_list"; -import TeamDropdown from "../common_components/team_dropdown"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { Tooltip } from "antd"; -import PremiumLoggingSettings from "../common_components/PremiumLoggingSettings"; -import Createuser from "../create_user_button"; -import debounce from "lodash/debounce"; -import { rolesWithWriteAccess } from "../../utils/roles"; -import BudgetDurationDropdown from "../common_components/budget_duration_dropdown"; import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { InfoCircleOutlined } from "@ant-design/icons"; +import { Accordion, AccordionBody, AccordionHeader, Button, Col, Grid, Text, TextInput, Title } from "@tremor/react"; +import { Button as Button2, Form, Input, Modal, Radio, Select, Switch, Tooltip } from "antd"; +import debounce from "lodash/debounce"; +import React, { useCallback, useEffect, useState } from "react"; +import { CopyToClipboard } from "react-copy-to-clipboard"; +import { rolesWithWriteAccess } from "../../utils/roles"; +import AgentSelector from "../agent_management/AgentSelector"; import { mapDisplayToInternalNames } from "../callback_info_helpers"; +import BudgetDurationDropdown from "../common_components/budget_duration_dropdown"; +import SchemaFormFields from "../common_components/check_openapi_schema"; +import KeyLifecycleSettings from "../common_components/KeyLifecycleSettings"; +import ModelAliasManager from "../common_components/ModelAliasManager"; +import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; +import PremiumLoggingSettings from "../common_components/PremiumLoggingSettings"; +import RateLimitTypeFormItem from "../common_components/RateLimitTypeFormItem"; +import TeamDropdown from "../common_components/team_dropdown"; +import Createuser from "../create_user_button"; +import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; +import { Team } from "../key_team_helpers/key_list"; import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; import MCPToolPermissions from "../mcp_server_management/MCPToolPermissions"; -import AgentSelector from "../agent_management/AgentSelector"; -import ModelAliasManager from "../common_components/ModelAliasManager"; import NotificationsManager from "../molecules/notifications_manager"; -import KeyLifecycleSettings from "../common_components/KeyLifecycleSettings"; -import RateLimitTypeFormItem from "../common_components/RateLimitTypeFormItem"; +import { + getGuardrailsList, + getPossibleUserRoles, + getPromptsList, + keyCreateCall, + keyCreateServiceAccountCall, + modelAvailableCall, + proxyBaseUrl, + userFilterUICall, +} from "../networking"; +import NumericalInput from "../shared/numerical_input"; +import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; const { Option } = Select; @@ -168,7 +165,6 @@ const CreateKey: React.FC = ({ const [userOptions, setUserOptions] = useState([]); const [userSearchLoading, setUserSearchLoading] = useState(false); const [mcpAccessGroups, setMcpAccessGroups] = useState([]); - const [mcpAccessGroupsLoaded, setMcpAccessGroupsLoaded] = useState(false); const [disabledCallbacks, setDisabledCallbacks] = useState([]); const [keyType, setKeyType] = useState("default"); const [modelAliases, setModelAliases] = useState<{ [key: string]: string }>({}); @@ -205,22 +201,6 @@ const CreateKey: React.FC = ({ } }, [accessToken, userID, userRole]); - const fetchMcpAccessGroups = async () => { - try { - if (accessToken == null) { - return; - } - const groups = await fetchMCPAccessGroups(accessToken); - setMcpAccessGroups(groups); - } catch (error) { - console.error("Failed to fetch MCP access groups:", error); - } - }; - - useEffect(() => { - fetchMcpAccessGroups(); - }, [accessToken]); - useEffect(() => { const fetchGuardrails = async () => { try { @@ -1053,15 +1033,7 @@ const CreateKey: React.FC = ({ options={predefinedTags} /> - { - if (!mcpAccessGroupsLoaded) { - fetchMcpAccessGroups(); - setMcpAccessGroupsLoaded(true); - } - }} - > + MCP Settings diff --git a/ui/litellm-dashboard/src/components/team/team_info.test.tsx b/ui/litellm-dashboard/src/components/team/team_info.test.tsx index 9b19611828f..4f7d70ba6ab 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.test.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.test.tsx @@ -1,5 +1,6 @@ import * as networking from "@/components/networking"; -import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { act, fireEvent, screen, waitFor } from "@testing-library/react"; +import { renderWithProviders } from "../../../tests/test-utils"; import { afterEach, describe, expect, it, vi } from "vitest"; import TeamInfoView from "./team_info"; @@ -62,7 +63,7 @@ describe("TeamInfoView", () => { vi.mocked(networking.getGuardrailsList).mockResolvedValue({ guardrails: [] }); vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); - render( + renderWithProviders( {}} @@ -124,7 +125,7 @@ describe("TeamInfoView", () => { vi.mocked(networking.getGuardrailsList).mockResolvedValue({ guardrails: [] }); vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); - render( + renderWithProviders( {}} @@ -219,7 +220,7 @@ describe("TeamInfoView", () => { vi.mocked(networking.getGuardrailsList).mockResolvedValue({ guardrails: [] }); vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); - render( + renderWithProviders( {}} @@ -310,7 +311,7 @@ describe("TeamInfoView", () => { vi.mocked(networking.getGuardrailsList).mockResolvedValue({ guardrails: [] }); vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); - render( + renderWithProviders( {}} @@ -373,7 +374,7 @@ describe("TeamInfoView", () => { vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: teamResponse.team_info, team_id: "123" } as any); - render( + renderWithProviders( {}} @@ -450,7 +451,7 @@ describe("TeamInfoView", () => { vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: teamResponse.team_info, team_id: "123" } as any); - render( + renderWithProviders( {}} diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index e23bc88dda8..85c6192693e 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -1,4 +1,5 @@ -import { render, waitFor } from "@testing-library/react"; +import { waitFor } from "@testing-library/react"; +import { renderWithProviders } from "../../../tests/test-utils"; import { describe, expect, it, vi } from "vitest"; import { KeyEditView } from "./key_edit_view"; import { KeyResponse } from "../key_team_helpers/key_list"; @@ -89,7 +90,7 @@ describe("KeyEditView", () => { key_rotation_at: undefined, }; it("should render", async () => { - const { getByText } = render( + const { getByText } = renderWithProviders( {}} @@ -107,7 +108,7 @@ describe("KeyEditView", () => { }); it("should render tags", async () => { - const { getByText } = render( + const { getByText } = renderWithProviders( {}} @@ -125,7 +126,7 @@ describe("KeyEditView", () => { }); it("should not render tags in metadata textarea", async () => { - const { getByLabelText } = render( + const { getByLabelText } = renderWithProviders( {}} diff --git a/ui/litellm-dashboard/tests/test-utils.tsx b/ui/litellm-dashboard/tests/test-utils.tsx index cf7fbaf0d8f..ed1f248648e 100644 --- a/ui/litellm-dashboard/tests/test-utils.tsx +++ b/ui/litellm-dashboard/tests/test-utils.tsx @@ -1,9 +1,26 @@ import React, { PropsWithChildren } from "react"; import { render, RenderOptions } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +// Create a client for testing +const queryClient = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + gcTime: Infinity, + staleTime: Infinity, + refetchOnWindowFocus: false, + refetchOnReconnect: false, + refetchOnMount: false, + }, + mutations: { + retry: false, + }, + }, +}); const Providers: React.FC = ({ children }) => { - // Add future providers here (Theme/Router/QueryClient/etc.) - return <>{children}; + return {children}; }; export const renderWithProviders = (ui: React.ReactElement, options?: RenderOptions) => From f258dbb03a6d967ba670944a352b7c87df86c82b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 22 Dec 2025 14:40:15 -0800 Subject: [PATCH 023/131] Fixing build --- .../components/mcp_server_management/MCPServerSelector.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx b/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx index f30a76a229b..7830edf5867 100644 --- a/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx +++ b/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx @@ -1,8 +1,7 @@ -import React from "react"; -import { Select } from "antd"; -import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"; import { useMCPAccessGroups } from "@/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups"; -import { MCPServer } from "../mcp_tools/types"; +import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"; +import { Select } from "antd"; +import React from "react"; interface MCPServerSelectorProps { onChange: (selected: { servers: string[]; accessGroups: string[] }) => void; From 78e3ae7bdeb115aa425de9481157fe0d5b123037 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 22 Dec 2025 15:17:36 -0800 Subject: [PATCH 024/131] Fix MCP Select button submitting form --- .../MCPToolPermissions.test.tsx | 133 +++++++++++++++--- .../MCPToolPermissions.tsx | 13 +- 2 files changed, 120 insertions(+), 26 deletions(-) diff --git a/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.test.tsx b/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.test.tsx index ec541d7a63c..fdd69d064d3 100644 --- a/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.test.tsx @@ -1,22 +1,12 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -import { render, screen, waitFor } from "@testing-library/react"; +import { screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { renderWithProviders } from "../../../tests/test-utils"; import MCPToolPermissions from "./MCPToolPermissions"; import * as networking from "../networking"; vi.mock("../networking"); -const createQueryClient = () => - new QueryClient({ - defaultOptions: { - queries: { - retry: false, - gcTime: 0, - }, - }, - }); - describe("MCPToolPermissions", () => { const mockAccessToken = "test-token"; const mockServerId = "server-123"; @@ -53,16 +43,13 @@ describe("MCPToolPermissions", () => { error: false, }); - const queryClient = createQueryClient(); - render( - - - , + renderWithProviders( + , ); // Wait for server and tools to load @@ -87,4 +74,106 @@ describe("MCPToolPermissions", () => { expect(networking.fetchMCPServers).toHaveBeenCalledWith(mockAccessToken); expect(networking.listMCPTools).toHaveBeenCalledWith(mockAccessToken, mockServerId); }); + + it("should select all tools when Select All button is clicked", async () => { + const mockOnChange = vi.fn(); + const mockTools = [ + { name: "read_wiki_structure", description: "Get documentation topics" }, + { name: "read_wiki_contents", description: "View documentation" }, + { name: "ask_question", description: "Ask questions" }, + ]; + + // Mock fetchMCPServers to return server details + vi.mocked(networking.fetchMCPServers).mockResolvedValue([ + { + server_id: mockServerId, + server_name: mockServerName, + alias: mockServerName, + }, + ]); + + // Mock listMCPTools to return tools for the server + vi.mocked(networking.listMCPTools).mockResolvedValue({ + tools: mockTools, + error: false, + }); + + renderWithProviders( + , + ); + + // Wait for server and tools to load + await waitFor(() => { + expect(screen.getByText(mockServerName)).toBeInTheDocument(); + }); + + await waitFor(() => { + expect(screen.getByText("read_wiki_structure")).toBeInTheDocument(); + }); + + // Click the Select All button + const selectAllButton = screen.getByRole("button", { name: "Select All" }); + await userEvent.click(selectAllButton); + + // Verify onChange was called with all tools selected + expect(mockOnChange).toHaveBeenCalledWith({ + [mockServerId]: ["read_wiki_structure", "read_wiki_contents", "ask_question"], + }); + }); + + it("should deselect all tools when Deselect All button is clicked", async () => { + const mockOnChange = vi.fn(); + const mockTools = [ + { name: "read_wiki_structure", description: "Get documentation topics" }, + { name: "read_wiki_contents", description: "View documentation" }, + { name: "ask_question", description: "Ask questions" }, + ]; + + // Mock fetchMCPServers to return server details + vi.mocked(networking.fetchMCPServers).mockResolvedValue([ + { + server_id: mockServerId, + server_name: mockServerName, + alias: mockServerName, + }, + ]); + + // Mock listMCPTools to return tools for the server + vi.mocked(networking.listMCPTools).mockResolvedValue({ + tools: mockTools, + error: false, + }); + + renderWithProviders( + , + ); + + // Wait for server and tools to load + await waitFor(() => { + expect(screen.getByText(mockServerName)).toBeInTheDocument(); + }); + + await waitFor(() => { + expect(screen.getByText("read_wiki_structure")).toBeInTheDocument(); + }); + + // Click the Deselect All button + const deselectAllButton = screen.getByRole("button", { name: "Deselect All" }); + await userEvent.click(deselectAllButton); + + // Verify onChange was called with no tools selected + expect(mockOnChange).toHaveBeenCalledWith({ + [mockServerId]: [], + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.tsx b/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.tsx index a567791e221..ec7e2797814 100644 --- a/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.tsx +++ b/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.tsx @@ -80,17 +80,19 @@ const MCPToolPermissions: React.FC = ({ const handleSelectAll = (serverId: string) => { const tools = serverTools[serverId] || []; - onChange({ + const newPermissions = { ...toolPermissions, [serverId]: tools.map((t) => t.name), - }); + }; + onChange(newPermissions); }; const handleDeselectAll = (serverId: string) => { - onChange({ + const newPermissions = { ...toolPermissions, [serverId]: [], - }); + }; + onChange(newPermissions); }; if (selectedServers.length === 0) { @@ -116,6 +118,7 @@ const MCPToolPermissions: React.FC = ({
diff --git a/ui/litellm-dashboard/src/components/model_dashboard/table.tsx b/ui/litellm-dashboard/src/components/model_dashboard/table.tsx index 344ff2e94f2..79224edba43 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/table.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/table.tsx @@ -3,10 +3,13 @@ import { flexRender, getCoreRowModel, getSortedRowModel, + getPaginationRowModel, SortingState, useReactTable, ColumnResizeMode, VisibilityState, + PaginationState, + OnChangeFn, } from "@tanstack/react-table"; import React from "react"; import { Table, TableHead, TableHeaderCell, TableBody, TableRow, TableCell } from "@tremor/react"; @@ -23,16 +26,20 @@ interface ModelDataTableProps { data: TData[]; columns: ColumnDef[]; isLoading?: boolean; - table: any; // Add table prop to access column visibility controls defaultSorting?: SortingState; + pagination?: PaginationState; + onPaginationChange?: OnChangeFn; + enablePagination?: boolean; } export function ModelDataTable({ data = [], columns, isLoading = false, - table, defaultSorting = [], + pagination, + onPaginationChange, + enablePagination = false, }: ModelDataTableProps) { const [sorting, setSorting] = React.useState(defaultSorting); const [columnResizeMode] = React.useState("onChange"); @@ -46,13 +53,16 @@ export function ModelDataTable({ sorting, columnSizing, columnVisibility, + ...(enablePagination && pagination ? { pagination } : {}), }, columnResizeMode, onSortingChange: setSorting, onColumnSizingChange: setColumnSizing, onColumnVisibilityChange: setColumnVisibility, + ...(enablePagination && onPaginationChange ? { onPaginationChange } : {}), getCoreRowModel: getCoreRowModel(), getSortedRowModel: getSortedRowModel(), + ...(enablePagination ? { getPaginationRowModel: getPaginationRowModel() } : {}), enableSorting: true, enableColumnResizing: true, defaultColumn: { @@ -61,13 +71,6 @@ export function ModelDataTable({ }, }); - // Expose table instance to parent - React.useEffect(() => { - if (table) { - table.current = tableInstance; - } - }, [tableInstance, table]); - const getHeaderText = (header: any): string => { if (typeof header === "string") { return header; From 5d1fe86cda3609071b29b4b3900cc58002184ff1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 14:33:27 -0800 Subject: [PATCH 081/131] Tests --- .../components/AllModelsTab.test.tsx | 103 ++++++++++++------ 1 file changed, 72 insertions(+), 31 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx index a4bb20128e0..dfa400e6ea9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx @@ -1,9 +1,27 @@ import * as useAuthorizedModule from "@/app/(dashboard)/hooks/useAuthorized"; -import * as useTeamsModule from "@/app/(dashboard)/hooks/useTeams"; import { render, screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import AllModelsTab from "./AllModelsTab"; +// Mock the useModelsInfo hook +const mockUseModelsInfo = vi.fn(() => ({ data: { data: [] } })) as any; + +vi.mock("../../hooks/models/useModels", () => ({ + useModelsInfo: () => mockUseModelsInfo(), +})); + +// Mock the useTeams hook (react-query implementation) +const mockUseTeams = vi.fn(() => ({ + data: [], + isLoading: false, + error: null, + refetch: vi.fn(), +})) as any; + +vi.mock("../../hooks/teams/useTeams", () => ({ + useTeams: () => mockUseTeams(), +})); + describe("AllModelsTab", () => { const mockSetSelectedModelGroup = vi.fn(); const mockSetSelectedModelId = vi.fn(); @@ -18,9 +36,6 @@ describe("AllModelsTab", () => { setSelectedModelId: mockSetSelectedModelId, setSelectedTeamId: mockSetSelectedTeamId, setEditModel: mockSetEditModel, - modelData: { - data: [], - }, }; const mockUseAuthorized = { @@ -40,9 +55,13 @@ describe("AllModelsTab", () => { }); it("should render with empty data", () => { - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: [], - setTeams: vi.fn(), + mockUseModelsInfo.mockReturnValueOnce({ data: { data: [] } }); + + mockUseTeams.mockReturnValueOnce({ + data: [], + isLoading: false, + error: null, + refetch: vi.fn(), }); render(); @@ -66,9 +85,11 @@ describe("AllModelsTab", () => { }, ]; - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: mockTeams, - setTeams: vi.fn(), + mockUseTeams.mockReturnValueOnce({ + data: mockTeams, + isLoading: false, + error: null, + refetch: vi.fn(), }); const modelData = { @@ -92,7 +113,9 @@ describe("AllModelsTab", () => { ], }; - render(); + mockUseModelsInfo.mockReturnValue({ data: modelData }); + + render(); await waitFor(() => { expect(screen.getByText("Showing 0 results")).toBeInTheDocument(); @@ -116,9 +139,11 @@ describe("AllModelsTab", () => { }, ]; - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: mockTeams, - setTeams: vi.fn(), + mockUseTeams.mockReturnValue({ + data: mockTeams, + isLoading: false, + error: null, + refetch: vi.fn(), }); const modelData = { @@ -142,7 +167,9 @@ describe("AllModelsTab", () => { ], }; - render(); + mockUseModelsInfo.mockReturnValue({ data: modelData }); + + render(); await waitFor(() => { expect(screen.getByText("Showing 0 results")).toBeInTheDocument(); @@ -150,9 +177,11 @@ describe("AllModelsTab", () => { }); it("should filter models by direct_access for personal team", async () => { - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: [], - setTeams: vi.fn(), + mockUseTeams.mockReturnValue({ + data: [], + isLoading: false, + error: null, + refetch: vi.fn(), }); const modelData = { @@ -178,7 +207,9 @@ describe("AllModelsTab", () => { ], }; - render(); + mockUseModelsInfo.mockReturnValue({ data: modelData }); + + render(); await waitFor(() => { expect(screen.getByText("Showing 1 - 1 of 1 results")).toBeInTheDocument(); @@ -186,9 +217,11 @@ describe("AllModelsTab", () => { }); it("should show config model status for models defined in configs", async () => { - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: [], - setTeams: vi.fn(), + mockUseTeams.mockReturnValue({ + data: [], + isLoading: false, + error: null, + refetch: vi.fn(), }); const modelData = { @@ -226,7 +259,9 @@ describe("AllModelsTab", () => { ], }; - render(); + mockUseModelsInfo.mockReturnValue({ data: modelData }); + + render(); await waitFor(() => { expect(screen.getByText("Config Model")).toBeInTheDocument(); @@ -235,19 +270,21 @@ describe("AllModelsTab", () => { }); it("should show 'Defined in config' for models defined in configs", async () => { - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: [], - setTeams: vi.fn(), + mockUseTeams.mockReturnValue({ + data: [], + isLoading: false, + error: null, + refetch: vi.fn(), }); const modelData = { data: [ { - model_name: "gpt-4-config-model", - litellm_model_name: "gpt-4-config-model", + model_name: "gpt-4-config", + litellm_model_name: "gpt-4-config", provider: "openai", model_info: { - id: "model-config-defined", + id: "model-config-1", db_model: false, direct_access: true, access_via_team_ids: [], @@ -260,8 +297,12 @@ describe("AllModelsTab", () => { ], }; - render(); + mockUseModelsInfo.mockReturnValue({ data: modelData }); - expect(screen.getByText("Defined in config")).toBeInTheDocument(); + render(); + + await waitFor(() => { + expect(screen.getByText("Defined in config")).toBeInTheDocument(); + }); }); }); From 4ce135727f8b00e6fa1af077be15b2e2accc5983 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 14:51:00 -0800 Subject: [PATCH 082/131] Fixing build --- .../model_dashboard/HealthCheckComponent.tsx | 1 - .../src/components/model_hub_table.tsx | 9 +- .../src/components/public_model_hub.tsx | 37 +++----- .../components/templates/model_dashboard.tsx | 93 ++++++++++--------- 4 files changed, 66 insertions(+), 74 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx index 994ea8adfc0..5d35b92684c 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx @@ -596,7 +596,6 @@ const HealthCheckComponent: React.FC = ({ }; })} isLoading={false} - table={healthTableRef} /> diff --git a/ui/litellm-dashboard/src/components/model_hub_table.tsx b/ui/litellm-dashboard/src/components/model_hub_table.tsx index 7d48bf68aed..f45e44ce905 100644 --- a/ui/litellm-dashboard/src/components/model_hub_table.tsx +++ b/ui/litellm-dashboard/src/components/model_hub_table.tsx @@ -1,10 +1,9 @@ import { CopyOutlined } from "@ant-design/icons"; -import { Table as TableInstance } from "@tanstack/react-table"; import { Badge, Button, Card, Tab, TabGroup, TabList, TabPanel, TabPanels, Text, Title } from "@tremor/react"; import { Modal } from "antd"; import { Copy } from "lucide-react"; import { useRouter } from "next/navigation"; -import React, { useCallback, useEffect, useRef, useState } from "react"; +import React, { useCallback, useEffect, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { isAdminRole } from "../utils/roles"; import { agentHubColumns, AgentHubData } from "./agent_hub_table_columns"; @@ -76,9 +75,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, const [isMcpModalVisible, setIsMcpModalVisible] = useState(false); const [isMakeMcpPublicModalVisible, setIsMakeMcpPublicModalVisible] = useState(false); const router = useRouter(); - const tableRef = useRef>(null); - const agentTableRef = useRef>(null); - const mcpTableRef = useRef>(null); useEffect(() => { const fetchData = async (accessToken: string) => { @@ -404,7 +400,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, columns={modelHubColumns(showModal, copyToClipboard, publicPage)} data={filteredData} isLoading={loading} - table={tableRef} defaultSorting={[{ id: "model_group", desc: false }]} /> @@ -431,7 +426,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, columns={agentHubColumns(showAgentModal, copyToClipboard, publicPage)} data={agentHubData || []} isLoading={agentLoading} - table={agentTableRef} defaultSorting={[{ id: "name", desc: false }]} /> @@ -458,7 +452,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, columns={mcpHubColumns(showMcpModal, copyToClipboard, publicPage)} data={mcpHubData || []} isLoading={mcpLoading} - table={mcpTableRef} defaultSorting={[{ id: "server_name", desc: false }]} /> diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 3493f0bf93f..4678dbe3f94 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -1,25 +1,24 @@ -import React, { useEffect, useState, useRef, useMemo } from "react"; -import { - modelHubPublicModelsCall, - getPublicModelHubInfo, - agentHubPublicModelsCall, - mcpHubPublicServersCall, - getUiConfig, -} from "./networking"; -import { ModelDataTable } from "./model_dashboard/table"; -import { ColumnDef } from "@tanstack/react-table"; -import { Card, Text, Title, Button } from "@tremor/react"; -import { Tag, Tooltip, Modal, Select, Tabs } from "antd"; +import { ThemeProvider } from "@/contexts/ThemeContext"; import { ExternalLinkIcon, SearchIcon } from "@heroicons/react/outline"; +import { ColumnDef } from "@tanstack/react-table"; +import { Button, Card, Text, Title } from "@tremor/react"; +import { Modal, Select, Tabs, Tag, Tooltip } from "antd"; import { Copy, Info } from "lucide-react"; -import { Table as TableInstance } from "@tanstack/react-table"; +import React, { useEffect, useMemo, useState } from "react"; +import { ModelDataTable } from "./model_dashboard/table"; +import NotificationsManager from "./molecules/notifications_manager"; +import Navbar from "./navbar"; +import { + agentHubPublicModelsCall, + getPublicModelHubInfo, + getUiConfig, + mcpHubPublicServersCall, + modelHubPublicModelsCall, +} from "./networking"; import { generateCodeSnippet } from "./playground/chat_ui/CodeSnippets"; import { getEndpointType } from "./playground/chat_ui/mode_endpoint_mapping"; import { MessageType } from "./playground/chat_ui/types"; import { getProviderLogoAndName } from "./provider_info_helpers"; -import Navbar from "./navbar"; -import { ThemeProvider } from "@/contexts/ThemeContext"; -import NotificationsManager from "./molecules/notifications_manager"; const { TabPane } = Tabs; @@ -118,9 +117,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded const [selectedMcpServer, setSelectedMcpServer] = useState(null); const [proxySettings, setProxySettings] = useState({}); const [activeTab, setActiveTab] = useState("models"); - const tableRef = useRef>(null); - const agentTableRef = useRef>(null); - const mcpTableRef = useRef>(null); useEffect(() => { const initializeAndFetch = async () => { @@ -1121,7 +1117,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded columns={publicModelHubColumns()} data={filteredData} isLoading={loading} - table={tableRef} defaultSorting={[{ id: "model_group", desc: false }]} /> @@ -1184,7 +1179,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded columns={publicAgentHubColumns()} data={filteredAgentData} isLoading={agentLoading} - table={agentTableRef} defaultSorting={[{ id: "name", desc: false }]} /> @@ -1248,7 +1242,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded columns={publicMCPHubColumns()} data={filteredMcpData} isLoading={mcpLoading} - table={mcpTableRef} defaultSorting={[{ id: "server_name", desc: false }]} /> diff --git a/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx index 6d43e0af057..9dbe04bffb1 100644 --- a/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx @@ -1,65 +1,74 @@ -import React, { useState, useEffect, useRef, useMemo } from "react"; import { Card, - Title, + Col, + Grid, Subtitle, Table, - TableHead, - TableRow, - TableHeaderCell, - TableCell, TableBody, + TableCell, + TableHead, + TableHeaderCell, + TableRow, Text, - Grid, - Col, + Title, } from "@tremor/react"; +import React, { useEffect, useMemo, useRef, useState } from "react"; import { CredentialItem, credentialListCall, CredentialsResponse } from "../networking"; import { handleAddModelSubmit } from "../add_model/handle_add_model_submit"; import CredentialsPanel from "@/components/model_add/credentials"; -import { getDisplayModelName } from "../view_model/model_name_display"; -import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react"; -import { Select, SelectItem, DateRangePickerValue } from "@tremor/react"; -import UsageDatePicker from "../shared/usage_date_picker"; +import { InfoCircleOutlined } from "@ant-design/icons"; +import { FilterIcon, RefreshIcon } from "@heroicons/react/outline"; +import { + AreaChart, + BarChart, + Button, + DateRangePickerValue, + Icon, + Select, + SelectItem, + Tab, + TabGroup, + TabList, + TabPanel, + TabPanels, +} from "@tremor/react"; +import type { UploadProps } from "antd"; +import { Form, InputNumber, Popover, Typography } from "antd"; +import AddModelTab from "../add_model/add_model_tab"; +import { Team } from "../key_team_helpers/key_list"; +import ModelInfoView from "../model_info_view"; +import TimeToFirstToken from "../model_metrics/time_to_first_token"; import { - modelInfoCall, - modelCostMap, - healthCheckCall, - modelMetricsCall, - streamingModelMetricsCall, - modelExceptionsCall, - modelMetricsSlowResponsesCall, - getCallbacksCall, - setCallbacksCall, - modelSettingsCall, adminGlobalActivityExceptions, adminGlobalActivityExceptionsPerDeployment, allEndUsersCall, + getCallbacksCall, + healthCheckCall, + modelCostMap, + modelExceptionsCall, + modelInfoCall, + modelMetricsCall, + modelMetricsSlowResponsesCall, + modelSettingsCall, + setCallbacksCall, + streamingModelMetricsCall, } from "../networking"; -import { BarChart, AreaChart } from "@tremor/react"; -import { Popover, Form, InputNumber } from "antd"; -import { Button } from "@tremor/react"; -import { Typography } from "antd"; -import { RefreshIcon, FilterIcon } from "@heroicons/react/outline"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import type { UploadProps } from "antd"; -import TimeToFirstToken from "../model_metrics/time_to_first_token"; -import { Team } from "../key_team_helpers/key_list"; +import { getPlaceholder, getProviderModels, provider_map, Providers } from "../provider_info_helpers"; +import UsageDatePicker from "../shared/usage_date_picker"; import TeamInfoView from "../team/team_info"; -import { Providers, provider_map, getPlaceholder, getProviderModels } from "../provider_info_helpers"; -import ModelInfoView from "../model_info_view"; -import AddModelTab from "../add_model/add_model_tab"; +import { getDisplayModelName } from "../view_model/model_name_display"; -import { ModelDataTable } from "../model_dashboard/table"; -import { columns } from "../molecules/models/columns"; -import PriceDataReload from "../price_data_reload"; -import HealthCheckComponent from "../model_dashboard/HealthCheckComponent"; -import PassThroughSettings from "../pass_through_settings"; -import ModelGroupAliasSettings from "../model_group_alias_settings"; import { all_admin_roles } from "@/utils/roles"; -import { Table as TableInstance, PaginationState } from "@tanstack/react-table"; +import { PaginationState } from "@tanstack/react-table"; +import HealthCheckComponent from "../model_dashboard/HealthCheckComponent"; +import { ModelDataTable } from "../model_dashboard/table"; +import ModelGroupAliasSettings from "../model_group_alias_settings"; +import { columns } from "../molecules/models/columns"; import NotificationsManager from "../molecules/notifications_manager"; +import PassThroughSettings from "../pass_through_settings"; +import PriceDataReload from "../price_data_reload"; interface ModelDashboardProps { accessToken: string | null; @@ -196,7 +205,6 @@ const OldModelDashboard: React.FC = ({ const [isDropdownOpen, setIsDropdownOpen] = useState(false); const [expandedRows, setExpandedRows] = useState>(new Set()); const dropdownRef = useRef(null); - const tableRef = useRef>(null); // Pagination state const [pagination, setPagination] = useState({ @@ -1325,7 +1333,6 @@ const OldModelDashboard: React.FC = ({ )} data={paginatedData} isLoading={false} - table={tableRef} /> From 85827aa217962d415ff72ffee41e650dfedb3a27 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 17:12:36 -0800 Subject: [PATCH 083/131] Resize columns working --- .../src/components/all_keys_table.tsx | 75 ++++++++++++++++++- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/all_keys_table.tsx b/ui/litellm-dashboard/src/components/all_keys_table.tsx index a915fe06179..210ce09fa34 100644 --- a/ui/litellm-dashboard/src/components/all_keys_table.tsx +++ b/ui/litellm-dashboard/src/components/all_keys_table.tsx @@ -1,6 +1,6 @@ "use client"; import React, { useEffect, useState } from "react"; -import { ColumnDef } from "@tanstack/react-table"; +import { ColumnDef, ColumnResizeMode, ColumnResizeDirection } from "@tanstack/react-table"; import { Select, SelectItem } from "@tremor/react"; import { Button } from "@tremor/react"; import KeyInfoView from "./templates/key_info_view"; @@ -125,6 +125,8 @@ export function AllKeysTable({ }: AllKeysTableProps) { const [selectedKeyId, setSelectedKeyId] = useState(null); const [userList, setUserList] = useState([]); + const [columnResizeMode, setColumnResizeMode] = React.useState("onChange"); + const [columnResizeDirection, setColumnResizeDirection] = React.useState("ltr"); const [sorting, setSorting] = React.useState(() => { if (currentSort) { return [ @@ -184,6 +186,7 @@ export function AllKeysTable({ { id: "expander", header: () => null, + size: 40, cell: ({ row }) => row.getCanExpand() ? (