From 1f83aafe3befe4438f73ebeaea22409cabc8bdbb Mon Sep 17 00:00:00 2001 From: Chesars Date: Sat, 31 Jan 2026 00:31:49 -0300 Subject: [PATCH 001/122] Add deprecation dates for xAI grok-2-vision-1212 and grok-3-mini models Per xAI announcement, these models will be deprecated on February 28, 2026: - xai/grok-2-vision-1212 - xai/grok-3-mini - xai/grok-3-mini-beta --- model_prices_and_context_window.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d958ea4503a..812e62c056a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -30354,6 +30354,7 @@ "supports_web_search": true }, "xai/grok-2-vision-1212": { + "deprecation_date": "2026-02-28", "input_cost_per_image": 2e-06, "input_cost_per_token": 2e-06, "litellm_provider": "xai", @@ -30452,6 +30453,7 @@ "supports_web_search": true }, "xai/grok-3-mini": { + "deprecation_date": "2026-02-28", "input_cost_per_token": 3e-07, "litellm_provider": "xai", "max_input_tokens": 131072, @@ -30467,6 +30469,7 @@ "supports_web_search": true }, "xai/grok-3-mini-beta": { + "deprecation_date": "2026-02-28", "input_cost_per_token": 3e-07, "litellm_provider": "xai", "max_input_tokens": 131072, From b417de6fe8ed389039e97a8807386cf48c34f934 Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 5 Feb 2026 17:44:01 -0300 Subject: [PATCH 002/122] feat(model_prices): add OpenRouter native models to model cost map Add openrouter/auto, openrouter/free, and openrouter/bodybuilder to the model prices and context window JSON. These are OpenRouter's own native models with their capabilities (context window, function calling, vision, reasoning, etc). --- model_prices_and_context_window.json | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 66c8e55c3ae..8803e6e7df6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24820,6 +24820,41 @@ "supports_prompt_caching": false, "supports_computer_use": false }, + "openrouter/openrouter/auto": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "openrouter", + "max_input_tokens": 2000000, + "max_tokens": 2000000, + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_audio_input": true + }, + "openrouter/openrouter/free": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_tokens": 200000, + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openrouter/bodybuilder": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "chat" + }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", From b7b48fd8ba789b4b60c5030890113b04e37e83e2 Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 5 Feb 2026 18:04:54 -0300 Subject: [PATCH 003/122] add supports_video_input for openrouter/auto --- model_prices_and_context_window.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8803e6e7df6..48ac68287b7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24832,7 +24832,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_vision": true, - "supports_audio_input": true + "supports_audio_input": true, + "supports_video_input": true }, "openrouter/openrouter/free": { "input_cost_per_token": 0, From 1d0c2f47a7b2291beb7fd7a45fa342f80ee455e0 Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 5 Feb 2026 18:33:02 -0300 Subject: [PATCH 004/122] docs: add Opus 4.6 to OpenRouter model map and update Anthropic docs - Add openrouter/anthropic/claude-opus-4.6 to model cost map - Update Bedrock docs to list Opus 4.6 and Sonnet 4.5 as 1M context compatible - Add missing Opus 4.6, Opus 4.5, and Opus 4.1 to Anthropic supported models table --- docs/my-website/docs/providers/anthropic.md | 4 ++++ docs/my-website/docs/providers/bedrock.md | 4 ++-- ...odel_prices_and_context_window_backup.json | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 446d663c5ac..ee04faaf7b3 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -4,6 +4,7 @@ import TabItem from '@theme/TabItem'; # Anthropic LiteLLM supports all anthropic models. +- `claude-opus-4-6-20260205` - `claude-sonnet-4-5-20250929` - `claude-opus-4-5-20251101` - `claude-opus-4-1-20250805` @@ -415,7 +416,10 @@ print(response) | Model Name | Function Call | |------------------|--------------------------------------------| +| claude-opus-4-6 | `completion('claude-opus-4-6-20260205', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-sonnet-4-5 | `completion('claude-sonnet-4-5-20250929', messages)` | `os.environ['ANTHROPIC_API_KEY']` | +| claude-opus-4-5 | `completion('claude-opus-4-5-20251101', messages)` | `os.environ['ANTHROPIC_API_KEY']` | +| claude-opus-4-1 | `completion('claude-opus-4-1-20250805', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-opus-4 | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-sonnet-4 | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-3.7 | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']` | diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index e546ed97656..bb07216a295 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -660,7 +660,7 @@ Same as [Anthropic API response](../providers/anthropic#usage---thinking--reason LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic-beta` header. This enables access to experimental features like: -- **1M Context Window** - Up to 1 million tokens of context (Claude Sonnet 4) +- **1M Context Window** - Up to 1 million tokens of context (Claude Opus 4.6, Sonnet 4.5, Sonnet 4) - **Computer Use Tools** - AI that can interact with computer interfaces - **Token-Efficient Tools** - More efficient tool usage patterns - **Extended Output** - Up to 128K output tokens @@ -670,7 +670,7 @@ LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic | Beta Feature | Header Value | Compatible Models | Description | |--------------|-------------|------------------|-------------| -| 1M Context Window | `context-1m-2025-08-07` | Claude Sonnet 4 | Enable 1 million token context window | +| 1M Context Window | `context-1m-2025-08-07` | Claude Opus 4.6, Sonnet 4.5, Sonnet 4 | Enable 1 million token context window | | Computer Use (Latest) | `computer-use-2025-01-24` | Claude 3.7 Sonnet | Latest computer use tools | | Computer Use (Legacy) | `computer-use-2024-10-22` | Claude 3.5 Sonnet v2 | Computer use tools for Claude 3.5 | | Token-Efficient Tools | `token-efficient-tools-2025-02-19` | Claude 3.7 Sonnet | More efficient tool usage | diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 66c8e55c3ae..186306b5b56 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23510,6 +23510,25 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "openrouter/anthropic/claude-opus-4.6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 3.75e-06, From 26d27803eb487a1b6b7881e2bdb74775b23c824b Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 19 Feb 2026 15:17:27 -0300 Subject: [PATCH 005/122] fix(models): disable function calling for PublicAI Apertus models The Apertus 8B and 70B models do not support standard OpenAI-style tool calling. Per Swiss AI's docs, tool use integration into inference engines is still in development. Set supports_function_calling and supports_tool_choice to false. Fixes #21124 --- model_prices_and_context_window.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 41acb5c8101..0ee2d586f85 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25907,8 +25907,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/swiss-ai/apertus-70b-instruct": { "input_cost_per_token": 0.0, @@ -25919,8 +25919,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/aisingapore/Gemma-SEA-LION-v4-27B-IT": { "input_cost_per_token": 0.0, From c6f60bed71f9198ed697c698bce533dbb9149106 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Mon, 23 Feb 2026 19:44:30 +0530 Subject: [PATCH 006/122] perf(spendlogs): optimize old spendlog deletion cron job --- .../litellm_proxy_extras/schema.prisma | 1 + .../db_transaction_queue/spend_log_cleanup.py | 30 ++++++------ litellm/proxy/schema.prisma | 1 + .../proxy/test_spend_log_cleanup.py | 47 ++++++------------- 4 files changed, 32 insertions(+), 47 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 4af7484148c..40bcaff67d6 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -500,6 +500,7 @@ model LiteLLM_SpendLogs { agent_id String? proxy_server_request Json? @default("{}") @@index([startTime]) + @@index([startTime, request_id]) @@index([end_user]) @@index([session_id]) } diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index 02fa84bae30..8c59c79ff0a 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -76,27 +76,29 @@ class SpendLogCleanup: "Max logs deleted - 1,00,000, rest of the logs will be deleted in next run" ) break - # Step 1: Find logs to delete - logs_to_delete = await prisma_client.db.litellm_spendlogs.find_many( - where={"startTime": {"lt": cutoff_date}}, - take=self.batch_size, + # Step 1: Find logs and delete them in one go without fetching to application + # Delete in batches, limited by self.batch_size + deleted_count = await prisma_client.db.execute_raw( + """ + DELETE FROM "LiteLLM_SpendLogs" + WHERE "request_id" IN ( + SELECT "request_id" FROM "LiteLLM_SpendLogs" + WHERE "startTime" < $1::timestamptz + LIMIT $2 + ) + """, + cutoff_date, + self.batch_size, ) - verbose_proxy_logger.info(f"Found {len(logs_to_delete)} logs in this batch") + verbose_proxy_logger.info(f"Deleted {deleted_count} logs in this batch") - if not logs_to_delete: + if deleted_count == 0: verbose_proxy_logger.info( f"No more logs to delete. Total deleted: {total_deleted}" ) break - request_ids = [log.request_id for log in logs_to_delete] - - # Step 2: Delete them in one go - await prisma_client.db.litellm_spendlogs.delete_many( - where={"request_id": {"in": request_ids}} - ) - - total_deleted += len(logs_to_delete) + total_deleted += deleted_count run_count += 1 # Add a small sleep to prevent overwhelming the database diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 50c0a55a875..7a1010d95c0 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -499,6 +499,7 @@ model LiteLLM_SpendLogs { agent_id String? proxy_server_request Json? @default("{}") @@index([startTime]) + @@index([startTime, request_id]) @@index([end_user]) @@index([session_id]) } diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py index 1ffbb83caef..c1fa3ad0c43 100644 --- a/tests/test_litellm/proxy/test_spend_log_cleanup.py +++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py @@ -151,28 +151,16 @@ async def test_should_delete_spend_logs(): @pytest.mark.asyncio async def test_cleanup_old_spend_logs_batch_deletion(): - from types import SimpleNamespace - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import AsyncMock, MagicMock # Setup Prisma client mock_prisma_client = MagicMock() mock_db = MagicMock() - # Mock spendlogs table - mock_spendlogs = MagicMock() - mock_spendlogs.find_many = AsyncMock() - mock_spendlogs.delete_many = AsyncMock() - - # Create 1500 mocked logs with .request_id - mock_logs = [SimpleNamespace(request_id=f"req_{i}") for i in range(1500)] - mock_spendlogs.find_many.side_effect = [ - mock_logs[:1000], # Batch 1 - mock_logs[1000:], # Batch 2 - [], # Done - ] + # Mock execute_raw to return deleted counts + mock_db.execute_raw = AsyncMock(side_effect=[1000, 500, 0]) # Wire up mocks - mock_db.litellm_spendlogs = mock_spendlogs mock_prisma_client.db = mock_db # Mock Redis cache and pod_lock_manager @@ -189,15 +177,13 @@ async def test_cleanup_old_spend_logs_batch_deletion(): assert cleaner._should_delete_spend_logs() is True await cleaner.cleanup_old_spend_logs(mock_prisma_client) - # Validate batching and deletion - assert mock_spendlogs.find_many.call_count == 3 - assert mock_spendlogs.delete_many.call_count == 2 - mock_spendlogs.delete_many.assert_any_call( - where={"request_id": {"in": [f"req_{i}" for i in range(1000)]}} - ) - mock_spendlogs.delete_many.assert_any_call( - where={"request_id": {"in": [f"req_{i}" for i in range(1000, 1500)]}} - ) + # Validate batching and deletion via raw SQL + assert mock_db.execute_raw.call_count == 3 + + # Check the first call argument + call_args_sql = mock_db.execute_raw.call_args_list[0][0][0] + assert 'DELETE FROM "LiteLLM_SpendLogs"' in call_args_sql + assert 'WHERE "request_id" IN' in call_args_sql @pytest.mark.asyncio @@ -208,10 +194,7 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff(): # Setup Prisma client mock_prisma_client = MagicMock() mock_db = MagicMock() - mock_spendlogs = MagicMock() - mock_spendlogs.find_many = AsyncMock(return_value=[]) - mock_spendlogs.delete_many = AsyncMock() - mock_db.litellm_spendlogs = mock_spendlogs + mock_db.execute_raw = AsyncMock(return_value=0) mock_prisma_client.db = mock_db # Mock Redis cache and pod_lock_manager @@ -229,7 +212,7 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff(): await cleaner.cleanup_old_spend_logs(mock_prisma_client) # Verify the cutoff date is correct - cutoff_date = mock_spendlogs.find_many.call_args[1]["where"]["startTime"]["lt"] + cutoff_date = mock_db.execute_raw.call_args[0][1] expected_cutoff = datetime.now(timezone.utc) - timedelta(seconds=86400) assert ( abs((cutoff_date - expected_cutoff).total_seconds()) < 1 @@ -242,14 +225,12 @@ async def test_cleanup_old_spend_logs_no_retention_period(): Test that no logs are deleted when no retention period is set """ mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_spendlogs.find_many = AsyncMock() - mock_prisma_client.db.litellm_spendlogs.delete = AsyncMock() + mock_prisma_client.db.execute_raw = AsyncMock() cleaner = SpendLogCleanup(general_settings={}) # no retention await cleaner.cleanup_old_spend_logs(mock_prisma_client) - mock_prisma_client.db.litellm_spendlogs.find_many.assert_not_called() - mock_prisma_client.db.litellm_spendlogs.delete.assert_not_called() + mock_prisma_client.db.execute_raw.assert_not_called() def test_cleanup_batch_size_env_var(monkeypatch): From a1c939b2ef74e72c74981d49e230b35957bbc4a9 Mon Sep 17 00:00:00 2001 From: Zero Clover Date: Wed, 25 Feb 2026 01:29:06 +0800 Subject: [PATCH 007/122] fix(image_generation): propagate extra_headers to OpenAI image generation Add headers parameter to image_generation() and aimage_generation() methods in OpenAI provider, and pass headers from images/main.py to ensure custom headers like cf-aig-authorization are properly forwarded to the OpenAI API. Aligns behavior with completion() method and Azure provider implementation. --- litellm/images/main.py | 1 + litellm/llms/openai/openai.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/images/main.py b/litellm/images/main.py index 6c4c502a7b0..236266af6ad 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -483,6 +483,7 @@ def image_generation( # noqa: PLR0915 organization=organization, aimg_generation=aimg_generation, client=client, + headers=headers, ) elif custom_llm_provider == "bedrock": if model is None: diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index c7524925bd0..7020f796bb7 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1401,6 +1401,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): client=None, max_retries=None, organization: Optional[str] = None, + headers: Optional[dict] = None, ): response = None try: @@ -1414,6 +1415,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): client=client, ) + if headers: + data["extra_headers"] = headers response = await openai_aclient.images.generate(**data, timeout=timeout) # type: ignore stringified_response = response.model_dump() ## LOGGING @@ -1446,6 +1449,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): client=None, aimg_generation=None, organization: Optional[str] = None, + headers: Optional[dict] = None, ) -> ImageResponse: data = {} try: @@ -1455,7 +1459,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): raise OpenAIError(status_code=422, message="max retries must be an int") if aimg_generation is True: - return self.aimage_generation(data=data, prompt=prompt, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries, organization=organization) # type: ignore + return self.aimage_generation(data=data, prompt=prompt, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries, organization=organization, headers=headers) # type: ignore openai_client: OpenAI = self._get_openai_client( # type: ignore is_async=False, @@ -1480,6 +1484,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): ) ## COMPLETION CALL + if headers: + data["extra_headers"] = headers _response = openai_client.images.generate(**data, timeout=timeout) # type: ignore response = _response.model_dump() From a5b4dfe6bee59f49657695d872cd48f007625138 Mon Sep 17 00:00:00 2001 From: Zero Clover Date: Wed, 25 Feb 2026 01:51:18 +0800 Subject: [PATCH 008/122] test(image_generation): add tests for extra_headers propagation Verify that extra_headers are correctly forwarded to OpenAI's images.generate() in both sync and async paths, and that they are absent when not provided. --- ...t_openai_image_generation_extra_headers.py | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py diff --git a/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py b/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py new file mode 100644 index 00000000000..33db9d33c1c --- /dev/null +++ b/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py @@ -0,0 +1,212 @@ +""" +Unit tests for extra_headers propagation in OpenAI image generation. + +Verifies that extra_headers passed to litellm.image_generation() / +litellm.aimage_generation() are forwarded to the OpenAI API client as +extra_headers in the images.generate() call. +""" + +import os +import sys +from unittest.mock import MagicMock, AsyncMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.openai.openai import OpenAIChatCompletion + + +@pytest.fixture +def openai_chat_completions(): + return OpenAIChatCompletion() + + +@pytest.fixture +def mock_logging_obj(): + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + logging_obj.post_call = MagicMock() + return logging_obj + + +class TestImageGenerationExtraHeaders: + """Test that extra_headers are properly injected into OpenAI image generation calls.""" + + def test_sync_image_generation_with_headers( + self, openai_chat_completions, mock_logging_obj + ): + """Sync image_generation should pass headers as extra_headers to images.generate().""" + mock_image_data = MagicMock() + mock_image_data.model_dump.return_value = { + "created": 1700000000, + "data": [{"url": "https://example.com/image.png"}], + } + + mock_openai_client = MagicMock() + mock_openai_client.images.generate.return_value = mock_image_data + mock_openai_client.api_key = "test-key" + mock_openai_client._base_url._uri_reference = "https://api.openai.com" + + test_headers = {"cf-aig-authorization": "Bearer custom-token"} + + openai_chat_completions.image_generation( + model="dall-e-3", + prompt="A white cat", + timeout=60.0, + optional_params={}, + logging_obj=mock_logging_obj, + api_key="test-key", + headers=test_headers, + client=mock_openai_client, + ) + + _, kwargs = mock_openai_client.images.generate.call_args + assert kwargs.get("extra_headers") == test_headers + + def test_sync_image_generation_without_headers( + self, openai_chat_completions, mock_logging_obj + ): + """Sync image_generation without headers should not inject extra_headers.""" + mock_image_data = MagicMock() + mock_image_data.model_dump.return_value = { + "created": 1700000000, + "data": [{"url": "https://example.com/image.png"}], + } + + mock_openai_client = MagicMock() + mock_openai_client.images.generate.return_value = mock_image_data + mock_openai_client.api_key = "test-key" + mock_openai_client._base_url._uri_reference = "https://api.openai.com" + + openai_chat_completions.image_generation( + model="dall-e-3", + prompt="A white cat", + timeout=60.0, + optional_params={}, + logging_obj=mock_logging_obj, + api_key="test-key", + client=mock_openai_client, + ) + + _, kwargs = mock_openai_client.images.generate.call_args + assert "extra_headers" not in kwargs + + @pytest.mark.asyncio + async def test_async_image_generation_with_headers( + self, openai_chat_completions, mock_logging_obj + ): + """Async aimage_generation should pass headers as extra_headers to images.generate().""" + mock_image_data = MagicMock() + mock_image_data.model_dump.return_value = { + "created": 1700000000, + "data": [{"url": "https://example.com/image.png"}], + } + + mock_openai_client = MagicMock() + mock_openai_client.images.generate = AsyncMock(return_value=mock_image_data) + mock_openai_client.api_key = "test-key" + + test_headers = {"cf-aig-authorization": "Bearer custom-token"} + + await openai_chat_completions.aimage_generation( + prompt="A white cat", + data={"model": "dall-e-3", "prompt": "A white cat"}, + model_response=MagicMock(), + timeout=60.0, + logging_obj=mock_logging_obj, + api_key="test-key", + headers=test_headers, + client=mock_openai_client, + ) + + _, kwargs = mock_openai_client.images.generate.call_args + assert kwargs.get("extra_headers") == test_headers + + @pytest.mark.asyncio + async def test_async_image_generation_without_headers( + self, openai_chat_completions, mock_logging_obj + ): + """Async aimage_generation without headers should not inject extra_headers.""" + mock_image_data = MagicMock() + mock_image_data.model_dump.return_value = { + "created": 1700000000, + "data": [{"url": "https://example.com/image.png"}], + } + + mock_openai_client = MagicMock() + mock_openai_client.images.generate = AsyncMock(return_value=mock_image_data) + mock_openai_client.api_key = "test-key" + + await openai_chat_completions.aimage_generation( + prompt="A white cat", + data={"model": "dall-e-3", "prompt": "A white cat"}, + model_response=MagicMock(), + timeout=60.0, + logging_obj=mock_logging_obj, + api_key="test-key", + client=mock_openai_client, + ) + + _, kwargs = mock_openai_client.images.generate.call_args + assert "extra_headers" not in kwargs + + def test_sync_image_generation_forwards_headers_to_async( + self, openai_chat_completions, mock_logging_obj + ): + """When aimg_generation=True, image_generation should forward headers to aimage_generation.""" + with patch.object( + openai_chat_completions, "aimage_generation" + ) as mock_aimage_gen: + mock_aimage_gen.return_value = MagicMock() + + test_headers = {"x-custom-header": "value"} + + openai_chat_completions.image_generation( + model="dall-e-3", + prompt="A white cat", + timeout=60.0, + optional_params={}, + logging_obj=mock_logging_obj, + api_key="test-key", + aimg_generation=True, + headers=test_headers, + ) + + mock_aimage_gen.assert_called_once() + call_kwargs = mock_aimage_gen.call_args[1] + assert call_kwargs["headers"] == test_headers + + +class TestImageGenerationEntryPointHeaders: + """Test that litellm.image_generation() passes headers through to the OpenAI provider.""" + + @pytest.mark.asyncio + async def test_extra_headers_reach_openai_provider(self): + """End-to-end: extra_headers from litellm.aimage_generation() reach OpenAI images.generate().""" + import litellm + + mock_image_data = MagicMock() + mock_image_data.model_dump.return_value = { + "created": 1700000000, + "data": [{"url": "https://example.com/image.png"}], + } + + mock_openai_client = MagicMock() + mock_openai_client.images.generate = AsyncMock(return_value=mock_image_data) + mock_openai_client.api_key = "test-key" + mock_openai_client._base_url._uri_reference = "https://api.openai.com" + + test_headers = {"cf-aig-authorization": "Bearer my-secret"} + + await litellm.aimage_generation( + model="dall-e-3", + prompt="A white cat", + extra_headers=test_headers, + client=mock_openai_client, + api_key="test-key", + ) + + mock_openai_client.images.generate.assert_called_once() + _, kwargs = mock_openai_client.images.generate.call_args + assert kwargs.get("extra_headers") == test_headers From f453427264866e038d9b714701fbce925e0fe6c6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 25 Feb 2026 17:19:33 +0530 Subject: [PATCH 009/122] Add v1 for anthropic responses transformation --- .../messages/handler.py | 59 ++- .../responses_adapters/__init__.py | 3 + .../responses_adapters/handler.py | 213 +++++++++ .../responses_adapters/streaming_iterator.py | 265 ++++++++++++ .../responses_adapters/transformation.py | 407 ++++++++++++++++++ ...erimental_pass_through_messages_handler.py | 62 +-- 6 files changed, 958 insertions(+), 51 deletions(-) create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 7e5a4f22a7f..6fe0fcd4fdf 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -25,8 +25,18 @@ from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler +from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler from .utils import AnthropicMessagesRequestUtils, mock_response +# Providers that are routed directly to the OpenAI Responses API instead of +# going through chat/completions. +_RESPONSES_API_PROVIDERS = frozenset({"openai", "azure", "azure_text"}) + + +def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool: + """Return True when the provider should use the Responses API path.""" + return custom_llm_provider in _RESPONSES_API_PROVIDERS + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -282,29 +292,34 @@ def anthropic_messages_handler( ) ) if anthropic_messages_provider_config is None: - # Handle non-Anthropic models using the adapter - return ( - LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( - max_tokens=max_tokens, - messages=messages, - model=model, - metadata=metadata, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - _is_async=is_async, - api_key=api_key, - api_base=api_base, - client=client, - custom_llm_provider=custom_llm_provider, - **kwargs, + # Route to Responses API for OpenAI / Azure, chat/completions for everything else. + _shared_kwargs = dict( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + _is_async=is_async, + api_key=api_key, + api_base=api_base, + client=client, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + if _should_route_to_responses_api(custom_llm_provider): + return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + **_shared_kwargs ) + return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( + **_shared_kwargs ) if custom_llm_provider is None: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py new file mode 100644 index 00000000000..6ad3c7b0164 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py @@ -0,0 +1,3 @@ +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter + +__all__ = ["LiteLLMAnthropicToResponsesAPIAdapter"] diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py new file mode 100644 index 00000000000..18dbabb1e14 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -0,0 +1,213 @@ +""" +Handler for the Anthropic v1/messages -> OpenAI Responses API path. + +Used when the target model is an OpenAI or Azure model. +""" + +from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union + +import litellm +from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, +) +from litellm.types.llms.openai import ResponsesAPIResponse + +from .streaming_iterator import AnthropicResponsesStreamWrapper +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter + +_ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() + + +def _build_responses_kwargs( + *, + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + extra_kwargs: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). + """ + # Build a typed AnthropicMessagesRequest for the adapter + request_data: Dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens} + if metadata: + request_data["metadata"] = metadata + if system: + request_data["system"] = system + if temperature is not None: + request_data["temperature"] = temperature + if thinking: + request_data["thinking"] = thinking + if tool_choice: + request_data["tool_choice"] = tool_choice + if tools: + request_data["tools"] = tools + if top_p is not None: + request_data["top_p"] = top_p + if output_format: + request_data["output_format"] = output_format + + anthropic_request = AnthropicMessagesRequest(**request_data) + responses_kwargs = _ADAPTER.translate_request(anthropic_request) + + if stream: + responses_kwargs["stream"] = True + + # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) + excluded = {"anthropic_messages"} + for key, value in (extra_kwargs or {}).items(): + if key == "litellm_logging_obj" and value is not None: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObject, + ) + from litellm.types.utils import CallTypes + + if isinstance(value, LiteLLMLoggingObject): + # Reclassify as acompletion so the success handler doesn't try to + # validate the Responses API event as an AnthropicResponse. + # (Mirrors the pattern used in LiteLLMMessagesToCompletionTransformationHandler.) + setattr(value, "call_type", CallTypes.acompletion.value) + responses_kwargs[key] = value + elif key not in excluded and key not in responses_kwargs and value is not None: + responses_kwargs[key] = value + + return responses_kwargs + + +class LiteLLMMessagesToResponsesAPIHandler: + """ + Handles Anthropic /v1/messages requests for OpenAI / Azure models by + calling litellm.responses() / litellm.aresponses() directly and translating + the response back to Anthropic format. + """ + + @staticmethod + async def async_anthropic_messages_handler( + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + **kwargs, + ) -> Union[AnthropicMessagesResponse, AsyncIterator]: + responses_kwargs = _build_responses_kwargs( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + result = await litellm.aresponses(**responses_kwargs) + + if stream: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + return wrapper.async_anthropic_sse_wrapper() + + if not isinstance(result, ResponsesAPIResponse): + raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") + + return _ADAPTER.translate_response(result) + + @staticmethod + def anthropic_messages_handler( + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + _is_async: bool = False, + **kwargs, + ) -> Union[ + AnthropicMessagesResponse, + AsyncIterator[Any], + Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]], + ]: + if _is_async: + return LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + **kwargs, + ) + + # Sync path + responses_kwargs = _build_responses_kwargs( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + result = litellm.responses(**responses_kwargs) + + if stream: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + return wrapper.async_anthropic_sse_wrapper() + + if not isinstance(result, ResponsesAPIResponse): + raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") + + return _ADAPTER.translate_response(result) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py new file mode 100644 index 00000000000..0e6268e82f3 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -0,0 +1,265 @@ +# What is this? +## Translates OpenAI call to Anthropic `/v1/messages` format +import json +import traceback +from collections import deque +from typing import Any, AsyncIterator, Dict + +from litellm import verbose_logger +from litellm._uuid import uuid + + +class AnthropicResponsesStreamWrapper: + """ + Wraps a Responses API streaming iterator and re-emits events in Anthropic SSE format. + + Responses API event flow (relevant subset): + response.created -> message_start + response.output_item.added -> content_block_start (if message/function_call) + response.output_text.delta -> content_block_delta (text_delta) + response.reasoning_summary_text.delta -> content_block_delta (thinking_delta) + response.function_call_arguments.delta -> content_block_delta (input_json_delta) + response.output_item.done -> content_block_stop + response.completed -> message_delta + message_stop + """ + + def __init__( + self, + responses_stream: Any, + model: str, + ) -> None: + self.responses_stream = responses_stream + self.model = model + self._message_id: str = f"msg_{uuid.uuid4()}" + self._current_block_index: int = -1 + # Map item_id -> content_block_index so we can stop the right block later + self._item_id_to_block_index: Dict[str, int] = {} + # Track open function_call items by item_id so we can emit tool_use start + self._pending_tool_ids: Dict[str, str] = {} # item_id -> call_id / name accumulator + self._sent_message_start = False + self._sent_message_stop = False + self._chunk_queue: deque = deque() + + def _make_message_start(self) -> Dict[str, Any]: + return { + "type": "message_start", + "message": { + "id": self._message_id, + "type": "message", + "role": "assistant", + "content": [], + "model": self.model, + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, + } + + def _next_block_index(self) -> int: + self._current_block_index += 1 + return self._current_block_index + + def _process_event(self, event: Any) -> None: + """Convert one Responses API event into zero or more Anthropic chunks queued for emission.""" + event_type = getattr(event, "type", None) + if event_type is None and isinstance(event, dict): + event_type = event.get("type") + + if event_type is None: + return + + # ---- message_start ---- + if event_type == "response.created": + self._sent_message_start = True + self._chunk_queue.append(self._make_message_start()) + return + + # ---- content_block_start for a new output message item ---- + if event_type == "response.output_item.added": + item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None) + if item is None: + return + item_type = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) + item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) + + if item_type == "message": + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": {"type": "text", "text": ""}, + }) + elif item_type == "function_call": + call_id = getattr(item, "call_id", None) or (item.get("call_id") if isinstance(item, dict) else None) or "" + name = getattr(item, "name", None) or (item.get("name") if isinstance(item, dict) else None) or "" + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._pending_tool_ids[item_id] = call_id + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": { + "type": "tool_use", + "id": call_id, + "name": name, + "input": {}, + }, + }) + elif item_type == "reasoning": + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": {"type": "thinking", "thinking": ""}, + }) + return + + # ---- text delta ---- + if event_type == "response.output_text.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "text_delta", "text": delta}, + }) + return + + # ---- reasoning summary text delta ---- + if event_type == "response.reasoning_summary_text.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "thinking_delta", "thinking": delta}, + }) + return + + # ---- function call arguments delta ---- + if event_type == "response.function_call_arguments.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "input_json_delta", "partial_json": delta}, + }) + return + + # ---- output item done -> content_block_stop ---- + if event_type == "response.output_item.done": + item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None) + item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_stop", + "index": block_idx, + }) + return + + # ---- response completed -> message_delta + message_stop ---- + if event_type in ("response.completed", "response.failed", "response.incomplete"): + response_obj = getattr(event, "response", None) or (event.get("response") if isinstance(event, dict) else None) + stop_reason = "end_turn" + input_tokens = 0 + output_tokens = 0 + cache_creation_tokens = 0 + cache_read_tokens = 0 + + if response_obj is not None: + status = getattr(response_obj, "status", None) + if status == "incomplete": + stop_reason = "max_tokens" + usage = getattr(response_obj, "usage", None) + if usage is not None: + input_tokens = getattr(usage, "input_tokens", 0) or 0 + output_tokens = getattr(usage, "output_tokens", 0) or 0 + cache_creation_tokens = getattr(usage, "input_tokens_details", None) + cache_read_tokens = getattr(usage, "output_tokens_details", None) + # Prefer direct cache fields if present + cache_creation_tokens = getattr(usage, "cache_creation_input_tokens", 0) or 0 + cache_read_tokens = getattr(usage, "cache_read_input_tokens", 0) or 0 + + # Check if tool_use was in the output to override stop_reason + if response_obj is not None: + output = getattr(response_obj, "output", []) or [] + for out_item in output: + out_type = getattr(out_item, "type", None) or (out_item.get("type") if isinstance(out_item, dict) else None) + if out_type == "function_call": + stop_reason = "tool_use" + break + + usage_delta: Dict[str, Any] = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } + if cache_creation_tokens: + usage_delta["cache_creation_input_tokens"] = cache_creation_tokens + if cache_read_tokens: + usage_delta["cache_read_input_tokens"] = cache_read_tokens + + self._chunk_queue.append({ + "type": "message_delta", + "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "usage": usage_delta, + }) + self._chunk_queue.append({"type": "message_stop"}) + self._sent_message_stop = True + return + + def __aiter__(self) -> "AnthropicResponsesStreamWrapper": + return self + + async def __anext__(self) -> Dict[str, Any]: + # Return any queued chunks first + if self._chunk_queue: + return self._chunk_queue.popleft() + + # Emit message_start if not yet done (fallback if response.created wasn't fired) + if not self._sent_message_start: + self._sent_message_start = True + self._chunk_queue.append(self._make_message_start()) + return self._chunk_queue.popleft() + + # Consume the upstream stream + try: + async for event in self.responses_stream: + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() + except StopAsyncIteration: + pass + except Exception as e: + verbose_logger.error( + f"AnthropicResponsesStreamWrapper error: {e}\n{traceback.format_exc()}" + ) + + # Drain any remaining queued chunks + if self._chunk_queue: + return self._chunk_queue.popleft() + + raise StopAsyncIteration + + async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]: + """Yield SSE-encoded bytes for each Anthropic event chunk.""" + async for chunk in self: + if isinstance(chunk, dict): + event_type: str = str(chunk.get("type", "message")) + payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n" + yield payload.encode() + else: + yield chunk diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py new file mode 100644 index 00000000000..a428e8f4e8f --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -0,0 +1,407 @@ +""" +Transformation layer: Anthropic /v1/messages <-> OpenAI Responses API. + +This module owns all format conversions for the direct v1/messages -> Responses API +path used for OpenAI and Azure models. +""" + +import json +from typing import Any, Dict, List, Optional, Union, cast + +from litellm.types.llms.anthropic import ( + AllAnthropicToolsValues, + AnthopicMessagesAssistantMessageParam, + AnthropicFinishReason, + AnthropicMessagesRequest, + AnthropicMessagesToolChoice, + AnthropicMessagesUserMessageParam, + AnthropicResponseContentBlockText, + AnthropicResponseContentBlockThinking, + AnthropicResponseContentBlockToolUse, +) +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, + AnthropicUsage, +) +from litellm.types.llms.openai import ResponsesAPIResponse + + +class LiteLLMAnthropicToResponsesAPIAdapter: + """ + Converts Anthropic /v1/messages requests to OpenAI Responses API format and + converts Responses API responses back to Anthropic format. + """ + + # ------------------------------------------------------------------ # + # Request translation: Anthropic -> Responses API # + # ------------------------------------------------------------------ # + + @staticmethod + def _translate_anthropic_image_source_to_url(source: dict) -> Optional[str]: + """Convert Anthropic image source to a URL string.""" + source_type = source.get("type") + if source_type == "base64": + media_type = source.get("media_type", "image/jpeg") + data = source.get("data", "") + return f"data:{media_type};base64,{data}" if data else None + elif source_type == "url": + return source.get("url") + return None + + def translate_messages_to_responses_input( + self, + messages: List[ + Union[ + AnthropicMessagesUserMessageParam, + AnthopicMessagesAssistantMessageParam, + ] + ], + ) -> List[Dict[str, Any]]: + """ + Convert Anthropic messages list to Responses API `input` items. + + Mapping: + user text -> message(role=user, input_text) + user image -> message(role=user, input_image) + user tool_result -> function_call_output + assistant text -> message(role=assistant, output_text) + assistant tool_use -> function_call + """ + input_items: List[Dict[str, Any]] = [] + + for m in messages: + role = m["role"] + content = m.get("content") + + if role == "user": + if isinstance(content, str): + input_items.append({ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": content}], + }) + elif isinstance(content, list): + user_parts: List[Dict[str, Any]] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + user_parts.append({"type": "input_text", "text": block.get("text", "")}) + elif btype == "image": + url = self._translate_anthropic_image_source_to_url(block.get("source", {})) + if url: + user_parts.append({"type": "input_image", "image_url": url}) + elif btype == "tool_result": + tool_use_id = block.get("tool_use_id", "") + inner = block.get("content") + if inner is None: + output_text = "" + elif isinstance(inner, str): + output_text = inner + elif isinstance(inner, list): + parts = [ + c.get("text", "") + for c in inner + if isinstance(c, dict) and c.get("type") == "text" + ] + output_text = "\n".join(parts) + else: + output_text = str(inner) + # tool_result is a top-level item, not inside the message + input_items.append({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": output_text, + }) + if user_parts: + input_items.append({ + "type": "message", + "role": "user", + "content": user_parts, + }) + + elif role == "assistant": + if isinstance(content, str): + input_items.append({ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": content}], + }) + elif isinstance(content, list): + asst_parts: List[Dict[str, Any]] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + asst_parts.append({"type": "output_text", "text": block.get("text", "")}) + elif btype == "tool_use": + # tool_use becomes a top-level function_call item + input_items.append({ + "type": "function_call", + "call_id": block.get("id", ""), + "name": block.get("name", ""), + "arguments": json.dumps(block.get("input", {})), + }) + elif btype == "thinking": + thinking_text = block.get("thinking", "") + if thinking_text: + asst_parts.append({"type": "output_text", "text": thinking_text}) + if asst_parts: + input_items.append({ + "type": "message", + "role": "assistant", + "content": asst_parts, + }) + + return input_items + + def translate_tools_to_responses_api( + self, + tools: List[AllAnthropicToolsValues], + ) -> List[Dict[str, Any]]: + """Convert Anthropic tool definitions to Responses API function tools.""" + result: List[Dict[str, Any]] = [] + for tool in tools: + tool_dict = cast(Dict[str, Any], tool) + tool_type = tool_dict.get("type", "") + tool_name = tool_dict.get("name", "") + # web_search tool + if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search": + result.append({"type": "web_search_preview"}) + continue + func_tool: Dict[str, Any] = {"type": "function", "name": tool_name} + if "description" in tool_dict: + func_tool["description"] = tool_dict["description"] + if "input_schema" in tool_dict: + func_tool["parameters"] = tool_dict["input_schema"] + result.append(func_tool) + return result + + @staticmethod + def translate_tool_choice_to_responses_api( + tool_choice: AnthropicMessagesToolChoice, + ) -> Dict[str, Any]: + """Convert Anthropic tool_choice to Responses API tool_choice.""" + tc_type = tool_choice.get("type") + if tc_type == "any": + return {"type": "required"} + elif tc_type == "tool": + return {"type": "function", "name": tool_choice.get("name", "")} + return {"type": "auto"} + + @staticmethod + def translate_thinking_to_reasoning(thinking: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """ + Convert Anthropic thinking param to Responses API reasoning param. + + thinking.budget_tokens maps to reasoning effort: + >= 10000 -> high, >= 5000 -> medium, >= 2000 -> low, < 2000 -> minimal + """ + if not isinstance(thinking, dict) or thinking.get("type") != "enabled": + return None + budget = thinking.get("budget_tokens", 0) + if budget >= 10000: + effort = "high" + elif budget >= 5000: + effort = "medium" + elif budget >= 2000: + effort = "low" + else: + effort = "minimal" + return {"effort": effort, "summary": "detailed"} + + def translate_request( + self, + anthropic_request: AnthropicMessagesRequest, + ) -> Dict[str, Any]: + """ + Translate a full Anthropic /v1/messages request dict to + litellm.responses() / litellm.aresponses() kwargs. + """ + model: str = anthropic_request["model"] + messages_list = cast( + List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]], + anthropic_request["messages"], + ) + + responses_kwargs: Dict[str, Any] = { + "model": model, + "input": self.translate_messages_to_responses_input(messages_list), + } + + # system -> instructions + system = anthropic_request.get("system") + if system: + if isinstance(system, str): + responses_kwargs["instructions"] = system + elif isinstance(system, list): + text_parts = [ + b.get("text", "") + for b in system + if isinstance(b, dict) and b.get("type") == "text" + ] + responses_kwargs["instructions"] = "\n".join(filter(None, text_parts)) + + # max_tokens -> max_output_tokens + max_tokens = anthropic_request.get("max_tokens") + if max_tokens: + responses_kwargs["max_output_tokens"] = max_tokens + + # temperature / top_p passed through + if "temperature" in anthropic_request: + responses_kwargs["temperature"] = anthropic_request["temperature"] + if "top_p" in anthropic_request: + responses_kwargs["top_p"] = anthropic_request["top_p"] + + # tools + tools = anthropic_request.get("tools") + if tools: + responses_kwargs["tools"] = self.translate_tools_to_responses_api( + cast(List[AllAnthropicToolsValues], tools) + ) + + # tool_choice + tool_choice = anthropic_request.get("tool_choice") + if tool_choice: + responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api( + cast(AnthropicMessagesToolChoice, tool_choice) + ) + + # thinking -> reasoning + thinking = anthropic_request.get("thinking") + if isinstance(thinking, dict): + reasoning = self.translate_thinking_to_reasoning(thinking) + if reasoning: + responses_kwargs["reasoning"] = reasoning + + # output_format -> text format + output_format = anthropic_request.get("output_format") + if isinstance(output_format, dict) and output_format.get("type") == "json_schema": + schema = output_format.get("schema") + if schema: + responses_kwargs["text"] = { + "format": { + "type": "json_schema", + "name": "structured_output", + "schema": schema, + "strict": True, + } + } + + # metadata user_id -> user + metadata = anthropic_request.get("metadata") + if isinstance(metadata, dict) and "user_id" in metadata: + responses_kwargs["user"] = metadata["user_id"] + + return responses_kwargs + + # ------------------------------------------------------------------ # + # Response translation: Responses API -> Anthropic # + # ------------------------------------------------------------------ # + + def translate_response( + self, + response: ResponsesAPIResponse, + ) -> AnthropicMessagesResponse: + """ + Translate an OpenAI ResponsesAPIResponse to AnthropicMessagesResponse. + """ + from openai.types.responses import ( + ResponseFunctionToolCall, + ResponseOutputMessage, + ResponseReasoningItem, + ) + + from litellm.types.llms.openai import ResponseAPIUsage + + content: List[Dict[str, Any]] = [] + stop_reason: AnthropicFinishReason = "end_turn" + + for item in response.output: + if isinstance(item, ResponseReasoningItem): + for summary in item.summary: + text = getattr(summary, "text", "") + if text: + content.append( + AnthropicResponseContentBlockThinking( + type="thinking", + thinking=text, + signature=None, + ).model_dump() + ) + + elif isinstance(item, ResponseOutputMessage): + for part in item.content: + if getattr(part, "type", None) == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=getattr(part, "text", "") + ).model_dump() + ) + + elif isinstance(item, ResponseFunctionToolCall): + try: + input_data = json.loads(item.arguments) if item.arguments else {} + except (json.JSONDecodeError, TypeError): + input_data = {} + content.append( + AnthropicResponseContentBlockToolUse( + type="tool_use", + id=item.call_id or item.id, + name=item.name, + input=input_data, + ).model_dump() + ) + stop_reason = "tool_use" + + elif isinstance(item, dict): + item_type = item.get("type") + if item_type == "message": + for part in item.get("content", []): + if isinstance(part, dict) and part.get("type") == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("text", "") + ).model_dump() + ) + elif item_type == "function_call": + try: + input_data = json.loads(item.get("arguments", "{}")) + except (json.JSONDecodeError, TypeError): + input_data = {} + content.append( + AnthropicResponseContentBlockToolUse( + type="tool_use", + id=item.get("call_id") or item.get("id", ""), + name=item.get("name", ""), + input=input_data, + ).model_dump() + ) + stop_reason = "tool_use" + + # status -> stop_reason override + if response.status == "incomplete": + stop_reason = "max_tokens" + + # usage + raw_usage: Optional[ResponseAPIUsage] = response.usage + input_tokens = int(getattr(raw_usage, "input_tokens", 0) or 0) + output_tokens = int(getattr(raw_usage, "output_tokens", 0) or 0) + + anthropic_usage = AnthropicUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + return AnthropicMessagesResponse( + id=response.id, + type="message", + role="assistant", + model=response.model or "unknown-model", + stop_sequence=None, + usage=anthropic_usage, # type: ignore + content=content, # type: ignore + stop_reason=stop_reason, + ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 77c74a7847e..c671d9b37b8 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -16,13 +16,14 @@ from litellm.types.utils import Delta, ModelResponse, StreamingChoices def test_anthropic_experimental_pass_through_messages_handler(): """ - Test that api key is passed to litellm.completion + Test that api key is passed to litellm.responses for OpenAI models. + OpenAI and Azure models are routed directly to the Responses API. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=100, @@ -32,19 +33,20 @@ def test_anthropic_experimental_pass_through_messages_handler(): ) except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" + mock_responses.assert_called_once() + assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): """ - Test that api key is passed to litellm.completion + Test that api key, api base, and extra kwargs are forwarded to litellm.responses for Azure models. + Azure models are routed directly to the Responses API. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=100, @@ -56,10 +58,10 @@ def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_an ) except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" - assert mock_completion.call_args.kwargs["api_base"] == "test-api-base" - assert mock_completion.call_args.kwargs["custom_key"] == "custom_value" + mock_responses.assert_called_once() + assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" + assert mock_responses.call_args.kwargs["api_base"] == "test-api-base" + assert mock_responses.call_args.kwargs["custom_key"] == "custom_value" def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): @@ -143,19 +145,19 @@ async def test_bedrock_converse_budget_tokens_preserved(): assert thinking_param.get("budget_tokens") == 1024, f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" -def test_openai_model_with_thinking_converts_to_reasoning_effort(): +def test_openai_model_with_thinking_converts_to_reasoning(): """ - Test that when using a non-Anthropic model (like OpenAI gpt-5.2) with thinking parameter, - the thinking is converted to reasoning_effort and NOT passed as thinking. - - This ensures we don't regress on issue #16052 where non-Anthropic models would fail - with UnsupportedParamsError when thinking was passed directly. + Test that when using an OpenAI model with thinking parameter, the thinking is + converted to a Responses API `reasoning` param (NOT passed as thinking). + + OpenAI models are routed directly to the Responses API, so we verify that + litellm.responses() is called with `reasoning` properly set. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=1024, @@ -170,20 +172,22 @@ def test_openai_model_with_thinking_converts_to_reasoning_effort(): except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - - call_kwargs = mock_completion.call_args.kwargs - - # Verify reasoning_effort is set (converted from thinking) - assert "reasoning_effort" in call_kwargs, "reasoning_effort should be passed to completion" + mock_responses.assert_called_once() - # reasoning_effort is transformed into a dict with effort and summary fields - expected_reasoning_effort = {"effort": "minimal", "summary": "detailed"} - assert call_kwargs["reasoning_effort"] == expected_reasoning_effort, \ - f"reasoning_effort should be {expected_reasoning_effort} for budget_tokens=1024, got {call_kwargs.get('reasoning_effort')}" + call_kwargs = mock_responses.call_args.kwargs - # Verify thinking is NOT passed (non-Claude model) - assert "thinking" not in call_kwargs, "thinking should NOT be passed for non-Claude models" + # Verify reasoning is set (converted from thinking) + assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses" + + # budget_tokens=1024 -> effort="minimal" (< 2000 threshold) + expected_reasoning = {"effort": "minimal", "summary": "detailed"} + assert call_kwargs["reasoning"] == expected_reasoning, ( + f"reasoning should be {expected_reasoning} for budget_tokens=1024, " + f"got {call_kwargs.get('reasoning')}" + ) + + # Verify thinking is NOT passed directly to the Responses API + assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses" class TestThinkingParameterTransformation: From 9ea100a9f73654f087e60c6e408a76e3559566b5 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Wed, 25 Feb 2026 17:20:41 -0800 Subject: [PATCH 010/122] feat: add prometheus multiprocess directory cleanup Adds cleanup utilities for PROMETHEUS_MULTIPROC_DIR to prevent unbounded RAM/disk growth from stale .db files in multi-worker setups. Three-part lifecycle aligned with upstream prometheus_client docs: 1. Startup: wipe entire directory before workers fork (clean slate) 2. Shutdown: mark_process_dead() for own PID (removes gauge_live* only) 3. Periodic (hourly): scan for dead PIDs and call mark_process_dead() Counter/histogram files are never individually deleted at runtime to avoid partial counter resets that cause false spikes in rate()/increase(). Also auto-creates PROMETHEUS_MULTIPROC_DIR when prometheus callback is configured with multiple workers and the env var is not already set. --- litellm/proxy/prometheus_cleanup.py | 168 +++++++++ litellm/proxy/proxy_cli.py | 54 +++ litellm/proxy/proxy_server.py | 34 ++ .../proxy/test_prometheus_cleanup.py | 339 ++++++++++++++++++ 4 files changed, 595 insertions(+) create mode 100644 litellm/proxy/prometheus_cleanup.py create mode 100644 tests/test_litellm/proxy/test_prometheus_cleanup.py diff --git a/litellm/proxy/prometheus_cleanup.py b/litellm/proxy/prometheus_cleanup.py new file mode 100644 index 00000000000..34c48de74fa --- /dev/null +++ b/litellm/proxy/prometheus_cleanup.py @@ -0,0 +1,168 @@ +""" +Prometheus multiprocess directory cleanup utilities. + +When running with multiple workers and PROMETHEUS_MULTIPROC_DIR set, +each worker creates memory-mapped .db files (e.g., counter_1234.db). +When workers die or restart, gauge_live* files for dead PIDs must be +cleaned up via mark_process_dead(). Counter and histogram files are +kept since they contain cumulative data needed for correct aggregation. +""" + +from __future__ import annotations + +import glob +import os +import re +from typing import Optional, Set + +from litellm._logging import verbose_proxy_logger + +_PID_PATTERN = re.compile(r"_(\d+)\.db$") + + +def _get_multiproc_dir() -> Optional[str]: + """Return the PROMETHEUS_MULTIPROC_DIR env var value, or None.""" + return os.environ.get("PROMETHEUS_MULTIPROC_DIR") or os.environ.get( + "prometheus_multiproc_dir" + ) + + +def _is_pid_alive(pid: int) -> bool: + """ + Check if a process with the given PID is alive. + + Uses os.kill(pid, 0) which doesn't send a signal but checks existence. + - ProcessLookupError: process does not exist (dead) + - PermissionError: process exists but we can't signal it (alive, conservative) + - OSError: other error, treat as alive (conservative) + """ + try: + os.kill(pid, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + # Process exists but we don't have permission to signal it + return True + except OSError: + # Conservative: treat unknown errors as alive + return True + + +def _extract_pids_from_dir(directory: str) -> Set[int]: + """ + Scan .db filenames in a directory and extract PIDs. + + Prometheus client creates files like: + - counter_1234.db + - histogram_1234.db + - gauge_livesum_1234.db + - gauge_liveall_1234.db + + Returns a set of integer PIDs found. + """ + pids: Set[int] = set() + try: + for filename in os.listdir(directory): + if not filename.endswith(".db"): + continue + match = _PID_PATTERN.search(filename) + if match: + pids.add(int(match.group(1))) + except FileNotFoundError: + pass + return pids + + +def wipe_directory(directory: str) -> None: + """ + Delete all .db files in the prometheus multiproc directory. + + Called once in the master process before workers fork. Per the + prometheus_client docs: "This directory must be wiped between + process runs (before startup is recommended)." + + Any .db files present at this point are stale from a previous run. + """ + files = glob.glob(os.path.join(directory, "*.db")) + for filepath in files: + try: + os.remove(filepath) + except OSError as e: + verbose_proxy_logger.warning( + f"Failed to delete stale prometheus file {filepath}: {e}" + ) + if files: + verbose_proxy_logger.info( + f"Prometheus cleanup: wiped {len(files)} stale .db files from {directory}" + ) + + +def cleanup_own_pid_files() -> None: + """ + Mark the current process as dead for prometheus multiproc cleanup. + + Called during per-worker shutdown. Uses mark_process_dead() which + only removes gauge_live* files — counter and histogram files are + preserved since they contain cumulative data needed for correct + aggregation until the directory is wiped on next startup. + """ + directory = _get_multiproc_dir() + if not directory or not os.path.isdir(directory): + return + + from prometheus_client import multiprocess + + pid = os.getpid() + try: + multiprocess.mark_process_dead(pid) + verbose_proxy_logger.info( + f"Prometheus cleanup: marked worker PID {pid} as dead" + ) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to mark worker PID {pid} as dead: {e}" + ) + + +def mark_dead_pids(skip_pid: Optional[int] = None) -> None: + """ + Scan the prometheus multiproc directory and call mark_process_dead() + for PIDs that no longer exist. + + Uses prometheus_client.multiprocess.mark_process_dead() which only + removes gauge_live* files — counter and histogram files are preserved + since they contain cumulative data needed for correct aggregation. + + Args: + skip_pid: PID to skip (typically os.getpid()). If None, skips + the current process's PID. + """ + directory = _get_multiproc_dir() + if not directory or not os.path.isdir(directory): + return + + if skip_pid is None: + skip_pid = os.getpid() + + pids = _extract_pids_from_dir(directory) + + from prometheus_client import multiprocess + + dead_pids = [] + for pid in pids: + if pid == skip_pid: + continue + if not _is_pid_alive(pid): + try: + multiprocess.mark_process_dead(pid) + dead_pids.append(pid) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to mark PID {pid} as dead: {e}" + ) + + if dead_pids: + verbose_proxy_logger.info( + f"Prometheus cleanup: marked {len(dead_pids)} dead PIDs: {dead_pids}" + ) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index e91447af895..1be4931aeef 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -314,6 +314,54 @@ class ProxyInitializationHelpers: return None # Let uvicorn choose the default loop on Windows return "uvloop" + @staticmethod + def _maybe_setup_prometheus_multiproc_dir( + num_workers: int, + litellm_settings: Optional[dict], + ) -> None: + """ + Auto-create PROMETHEUS_MULTIPROC_DIR when running with multiple workers + and prometheus is configured as a callback. + + If the env var is already set by the user, just ensure the directory exists. + Otherwise, create a temp directory and set the env var. + """ + import tempfile + + if num_workers <= 1 or litellm_settings is None: + return + + # Check if prometheus is in any callback list + callbacks = litellm_settings.get("callbacks") or [] + success_callbacks = litellm_settings.get("success_callback") or [] + failure_callbacks = litellm_settings.get("failure_callback") or [] + all_callbacks = callbacks + success_callbacks + failure_callbacks + if "prometheus" not in all_callbacks: + return + + from litellm.proxy.prometheus_cleanup import wipe_directory + + existing_dir = os.environ.get( + "PROMETHEUS_MULTIPROC_DIR" + ) or os.environ.get("prometheus_multiproc_dir") + if existing_dir: + os.makedirs(existing_dir, exist_ok=True) + wipe_directory(existing_dir) + print( # noqa + f"LiteLLM: Using existing PROMETHEUS_MULTIPROC_DIR={existing_dir}" + ) + return + + multiproc_dir = os.path.join( + tempfile.gettempdir(), "litellm_prometheus_multiproc" + ) + os.makedirs(multiproc_dir, exist_ok=True) + wipe_directory(multiproc_dir) + os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir + print( # noqa + f"LiteLLM: Auto-created PROMETHEUS_MULTIPROC_DIR={multiproc_dir}" + ) + @click.command() @click.option( @@ -819,6 +867,12 @@ def run_server( # noqa: PLR0915 # DO NOT DELETE - enables global variables to work across files from litellm.proxy.proxy_server import app # noqa + # Auto-create PROMETHEUS_MULTIPROC_DIR for multi-worker setups + ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=num_workers, + litellm_settings=litellm_settings if config else None, + ) + # --- SEPARATE HEALTH APP LOGIC --- # To run the health app separately, use: # uvicorn litellm.proxy.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=4001 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f0b1e66818c..c24942e1356 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -709,6 +709,14 @@ async def proxy_shutdown_event(): # [DO NOT BLOCK shutdown events for this] pass + # Clean up this worker's prometheus multiproc .db files + try: + from litellm.proxy.prometheus_cleanup import cleanup_own_pid_files + + cleanup_own_pid_files() + except Exception as e: + verbose_proxy_logger.warning(f"Error cleaning up prometheus files: {e}") + ## RESET CUSTOM VARIABLES ## cleanup_router_config_variables() @@ -891,6 +899,12 @@ async def proxy_startup_event(app: FastAPI): # noqa: PLR0915 ## Initialize shared aiohttp session for connection reuse shared_aiohttp_session = await _initialize_shared_aiohttp_session() + ## Start periodic prometheus multiproc directory cleanup + from litellm.proxy.prometheus_cleanup import _get_multiproc_dir + + if _get_multiproc_dir(): + asyncio.create_task(_periodic_prometheus_cleanup()) + # End of startup event yield @@ -2045,6 +2059,26 @@ def _schedule_background_health_check_db_save( ) +async def _periodic_prometheus_cleanup(): + """ + Periodically mark dead worker PIDs in the prometheus multiproc directory. + + Uses mark_process_dead() which only removes gauge_live* files, preserving + counter/histogram data for correct aggregation. First run is 1 hour after + startup (startup wipe handles stale files), then every hour thereafter. + """ + from litellm.proxy.prometheus_cleanup import mark_dead_pids + + while True: + await asyncio.sleep(3600) # 1 hour + try: + mark_dead_pids() + except Exception as e: + verbose_proxy_logger.warning( + f"Error in periodic prometheus cleanup: {e}" + ) + + async def _run_background_health_check(): """ Periodically run health checks in the background on the endpoints. diff --git a/tests/test_litellm/proxy/test_prometheus_cleanup.py b/tests/test_litellm/proxy/test_prometheus_cleanup.py new file mode 100644 index 00000000000..9f69402b5b2 --- /dev/null +++ b/tests/test_litellm/proxy/test_prometheus_cleanup.py @@ -0,0 +1,339 @@ +""" +Tests for litellm.proxy.prometheus_cleanup module and +ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir. +""" + +from __future__ import annotations + +import os +import tempfile +from unittest.mock import patch + +import pytest + +from litellm.proxy.prometheus_cleanup import ( + _extract_pids_from_dir, + _is_pid_alive, + cleanup_own_pid_files, + mark_dead_pids, + wipe_directory, +) +from litellm.proxy.proxy_cli import ProxyInitializationHelpers + + +class TestExtractPidsFromDir: + def test_counter_files(self, tmp_path): + (tmp_path / "counter_1234.db").touch() + (tmp_path / "counter_5678.db").touch() + assert _extract_pids_from_dir(str(tmp_path)) == {1234, 5678} + + def test_histogram_files(self, tmp_path): + (tmp_path / "histogram_1234.db").touch() + assert _extract_pids_from_dir(str(tmp_path)) == {1234} + + def test_gauge_files(self, tmp_path): + (tmp_path / "gauge_livesum_1234.db").touch() + (tmp_path / "gauge_liveall_5678.db").touch() + assert _extract_pids_from_dir(str(tmp_path)) == {1234, 5678} + + def test_multiple_pids(self, tmp_path): + (tmp_path / "counter_100.db").touch() + (tmp_path / "histogram_200.db").touch() + (tmp_path / "gauge_livesum_300.db").touch() + assert _extract_pids_from_dir(str(tmp_path)) == {100, 200, 300} + + def test_non_db_files_ignored(self, tmp_path): + (tmp_path / "counter_1234.db").touch() + (tmp_path / "readme.txt").touch() + (tmp_path / "data.json").touch() + assert _extract_pids_from_dir(str(tmp_path)) == {1234} + + def test_empty_directory(self, tmp_path): + assert _extract_pids_from_dir(str(tmp_path)) == set() + + def test_nonexistent_directory(self): + assert _extract_pids_from_dir("/nonexistent/path/abc123") == set() + + def test_malformed_filenames(self, tmp_path): + (tmp_path / "counter_.db").touch() # no PID + (tmp_path / "random.db").touch() # no underscore+PID pattern + (tmp_path / "counter_abc.db").touch() # non-numeric PID + assert _extract_pids_from_dir(str(tmp_path)) == set() + + +class TestIsPidAlive: + def test_own_pid_is_alive(self): + assert _is_pid_alive(os.getpid()) is True + + def test_dead_pid(self): + # A very high PID is almost certainly dead + assert _is_pid_alive(4_000_000) is False + + def test_permission_error_treated_as_alive(self): + with patch("os.kill", side_effect=PermissionError): + assert _is_pid_alive(99999) is True + + +class TestWipeDirectory: + def test_deletes_all_db_files(self, tmp_path): + (tmp_path / "counter_1234.db").touch() + (tmp_path / "histogram_5678.db").touch() + (tmp_path / "gauge_livesum_9999.db").touch() + wipe_directory(str(tmp_path)) + assert not list(tmp_path.glob("*.db")) + + def test_preserves_non_db_files(self, tmp_path): + (tmp_path / "counter_1234.db").touch() + (tmp_path / "readme.txt").touch() + (tmp_path / "config.json").touch() + wipe_directory(str(tmp_path)) + assert not list(tmp_path.glob("*.db")) + assert (tmp_path / "readme.txt").exists() + assert (tmp_path / "config.json").exists() + + def test_empty_directory(self, tmp_path): + wipe_directory(str(tmp_path)) + assert not list(tmp_path.glob("*.db")) + + +class TestCleanupOwnPidFiles: + def test_calls_mark_process_dead_for_own_pid(self, tmp_path): + """Should call mark_process_dead with current PID on shutdown.""" + pid = os.getpid() + + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark_dead: + cleanup_own_pid_files() + mock_mark_dead.assert_called_once_with(pid) + + def test_noop_when_not_configured(self, tmp_path): + """Should not call mark_process_dead when env var is not set.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + os.environ.pop("prometheus_multiproc_dir", None) + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark_dead: + cleanup_own_pid_files() + mock_mark_dead.assert_not_called() + + +class TestMarkDeadPids: + def test_calls_mark_process_dead_for_dead_pids(self, tmp_path): + """mark_dead_pids should call mark_process_dead() for dead PIDs.""" + dead_pid = 4_000_000 + (tmp_path / f"counter_{dead_pid}.db").touch() + (tmp_path / f"gauge_livesum_{dead_pid}.db").touch() + + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark_dead: + mark_dead_pids() + mock_mark_dead.assert_called_once_with(dead_pid) + + def test_skips_own_pid(self, tmp_path): + """Should not call mark_process_dead for the current process.""" + own_pid = os.getpid() + (tmp_path / f"counter_{own_pid}.db").touch() + + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark_dead: + mark_dead_pids() + mock_mark_dead.assert_not_called() + + def test_skips_alive_pids(self, tmp_path): + """Should not call mark_process_dead for alive PIDs.""" + alive_pid = 99999 + (tmp_path / f"counter_{alive_pid}.db").touch() + + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch( + "litellm.proxy.prometheus_cleanup._is_pid_alive", + return_value=True, + ): + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark_dead: + mark_dead_pids() + mock_mark_dead.assert_not_called() + + def test_handles_mixed_alive_and_dead(self, tmp_path): + """Should only call mark_process_dead for dead PIDs.""" + dead_pid = 4_000_000 + own_pid = os.getpid() + (tmp_path / f"counter_{dead_pid}.db").touch() + (tmp_path / f"counter_{own_pid}.db").touch() + + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark_dead: + mark_dead_pids() + mock_mark_dead.assert_called_once_with(dead_pid) + + def test_handles_malformed_filenames(self, tmp_path): + """Malformed filenames should be ignored, dead PIDs still cleaned.""" + dead_pid = 4_000_000 + (tmp_path / "counter_.db").touch() + (tmp_path / "random.db").touch() + (tmp_path / f"counter_{dead_pid}.db").touch() + + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark_dead: + mark_dead_pids() + mock_mark_dead.assert_called_once_with(dead_pid) + + def test_permission_error_treated_as_alive(self, tmp_path): + """PermissionError from os.kill means process is alive, skip it.""" + target_pid = 99999 + (tmp_path / f"counter_{target_pid}.db").touch() + + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch("os.kill", side_effect=PermissionError): + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark_dead: + mark_dead_pids() + mock_mark_dead.assert_not_called() + + def test_noop_when_not_configured(self, tmp_path): + """Should do nothing when PROMETHEUS_MULTIPROC_DIR is not set.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + os.environ.pop("prometheus_multiproc_dir", None) + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark_dead: + mark_dead_pids() + mock_mark_dead.assert_not_called() + + +class TestMaybeSetupPrometheusMultiprocDir: + def test_auto_creates_dir_when_prometheus_configured(self): + """When multiple workers + prometheus callback, auto-creates temp dir.""" + litellm_settings = {"callbacks": ["prometheus"]} + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + os.environ.pop("prometheus_multiproc_dir", None) + + ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=4, + litellm_settings=litellm_settings, + ) + + result_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR") + assert result_dir is not None + assert os.path.isdir(result_dir) + expected = os.path.join( + tempfile.gettempdir(), "litellm_prometheus_multiproc" + ) + assert result_dir == expected + + # Cleanup + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + + def test_respects_existing_env_var(self, tmp_path): + """When PROMETHEUS_MULTIPROC_DIR is already set, don't override it.""" + custom_dir = str(tmp_path / "custom_prom") + litellm_settings = {"callbacks": ["prometheus"]} + + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": custom_dir}): + ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=4, + litellm_settings=litellm_settings, + ) + + assert os.environ["PROMETHEUS_MULTIPROC_DIR"] == custom_dir + assert os.path.isdir(custom_dir) + + def test_wipes_stale_files_on_setup(self, tmp_path): + """Should wipe existing .db files from a previous run.""" + custom_dir = str(tmp_path / "prom_dir") + os.makedirs(custom_dir) + # Simulate stale files from a previous run + for name in ["counter_9999.db", "histogram_9999.db", "gauge_livesum_9999.db"]: + with open(os.path.join(custom_dir, name), "w") as f: + f.write("stale") + + litellm_settings = {"callbacks": ["prometheus"]} + + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": custom_dir}): + ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=4, + litellm_settings=litellm_settings, + ) + + # All .db files should be wiped + import glob + remaining = glob.glob(os.path.join(custom_dir, "*.db")) + assert remaining == [] + + def test_noop_for_single_worker(self): + """Single worker doesn't need multiproc dir.""" + litellm_settings = {"callbacks": ["prometheus"]} + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + os.environ.pop("prometheus_multiproc_dir", None) + + ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=1, + litellm_settings=litellm_settings, + ) + + assert os.environ.get("PROMETHEUS_MULTIPROC_DIR") is None + + def test_noop_without_prometheus_callback(self): + """No prometheus callback = no setup needed.""" + litellm_settings = {"callbacks": ["langfuse"]} + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + os.environ.pop("prometheus_multiproc_dir", None) + + ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=4, + litellm_settings=litellm_settings, + ) + + assert os.environ.get("PROMETHEUS_MULTIPROC_DIR") is None + + def test_noop_with_none_litellm_settings(self): + """None litellm_settings = no setup needed.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + + ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=4, + litellm_settings=None, + ) + + assert os.environ.get("PROMETHEUS_MULTIPROC_DIR") is None + + def test_prometheus_in_success_callback(self): + """Prometheus in success_callback should also trigger setup.""" + litellm_settings = {"success_callback": ["prometheus"]} + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + os.environ.pop("prometheus_multiproc_dir", None) + + ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=2, + litellm_settings=litellm_settings, + ) + + result_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR") + assert result_dir is not None + assert os.path.isdir(result_dir) + + # Cleanup + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) From 9157c1ebe2445136dde2c416755699ae3155d644 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Wed, 25 Feb 2026 17:29:17 -0800 Subject: [PATCH 011/122] test: redundant tests --- .../proxy/test_prometheus_cleanup.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/tests/test_litellm/proxy/test_prometheus_cleanup.py b/tests/test_litellm/proxy/test_prometheus_cleanup.py index 9f69402b5b2..bb03d270b76 100644 --- a/tests/test_litellm/proxy/test_prometheus_cleanup.py +++ b/tests/test_litellm/proxy/test_prometheus_cleanup.py @@ -176,33 +176,6 @@ class TestMarkDeadPids: mark_dead_pids() mock_mark_dead.assert_called_once_with(dead_pid) - def test_handles_malformed_filenames(self, tmp_path): - """Malformed filenames should be ignored, dead PIDs still cleaned.""" - dead_pid = 4_000_000 - (tmp_path / "counter_.db").touch() - (tmp_path / "random.db").touch() - (tmp_path / f"counter_{dead_pid}.db").touch() - - with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): - with patch( - "prometheus_client.multiprocess.mark_process_dead" - ) as mock_mark_dead: - mark_dead_pids() - mock_mark_dead.assert_called_once_with(dead_pid) - - def test_permission_error_treated_as_alive(self, tmp_path): - """PermissionError from os.kill means process is alive, skip it.""" - target_pid = 99999 - (tmp_path / f"counter_{target_pid}.db").touch() - - with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): - with patch("os.kill", side_effect=PermissionError): - with patch( - "prometheus_client.multiprocess.mark_process_dead" - ) as mock_mark_dead: - mark_dead_pids() - mock_mark_dead.assert_not_called() - def test_noop_when_not_configured(self, tmp_path): """Should do nothing when PROMETHEUS_MULTIPROC_DIR is not set.""" with patch.dict(os.environ, {}, clear=False): From bfceefbb2614a54272dde43cc8e38a3a1fbc1965 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Wed, 25 Feb 2026 17:30:59 -0800 Subject: [PATCH 012/122] parameterize test --- .../proxy/test_prometheus_cleanup.py | 42 +++++-------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/tests/test_litellm/proxy/test_prometheus_cleanup.py b/tests/test_litellm/proxy/test_prometheus_cleanup.py index bb03d270b76..fbdb7ef4ac3 100644 --- a/tests/test_litellm/proxy/test_prometheus_cleanup.py +++ b/tests/test_litellm/proxy/test_prometheus_cleanup.py @@ -249,48 +249,26 @@ class TestMaybeSetupPrometheusMultiprocDir: remaining = glob.glob(os.path.join(custom_dir, "*.db")) assert remaining == [] - def test_noop_for_single_worker(self): - """Single worker doesn't need multiproc dir.""" - litellm_settings = {"callbacks": ["prometheus"]} - + @pytest.mark.parametrize( + "num_workers, litellm_settings", + [ + (1, {"callbacks": ["prometheus"]}), + (4, {"callbacks": ["langfuse"]}), + (4, None), + ], + ) + def test_noop_when_setup_not_needed(self, num_workers, litellm_settings): with patch.dict(os.environ, {}, clear=False): os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) os.environ.pop("prometheus_multiproc_dir", None) ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( - num_workers=1, + num_workers=num_workers, litellm_settings=litellm_settings, ) assert os.environ.get("PROMETHEUS_MULTIPROC_DIR") is None - def test_noop_without_prometheus_callback(self): - """No prometheus callback = no setup needed.""" - litellm_settings = {"callbacks": ["langfuse"]} - - with patch.dict(os.environ, {}, clear=False): - os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) - os.environ.pop("prometheus_multiproc_dir", None) - - ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( - num_workers=4, - litellm_settings=litellm_settings, - ) - - assert os.environ.get("PROMETHEUS_MULTIPROC_DIR") is None - - def test_noop_with_none_litellm_settings(self): - """None litellm_settings = no setup needed.""" - with patch.dict(os.environ, {}, clear=False): - os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) - - ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( - num_workers=4, - litellm_settings=None, - ) - - assert os.environ.get("PROMETHEUS_MULTIPROC_DIR") is None - def test_prometheus_in_success_callback(self): """Prometheus in success_callback should also trigger setup.""" litellm_settings = {"success_callback": ["prometheus"]} From 280464412d8c394d0fbbb5c9b101790d08f23ff8 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Wed, 25 Feb 2026 17:33:44 -0800 Subject: [PATCH 013/122] clean up docstring --- litellm/proxy/prometheus_cleanup.py | 66 ++++------------------------- 1 file changed, 8 insertions(+), 58 deletions(-) diff --git a/litellm/proxy/prometheus_cleanup.py b/litellm/proxy/prometheus_cleanup.py index 34c48de74fa..4dec57fa806 100644 --- a/litellm/proxy/prometheus_cleanup.py +++ b/litellm/proxy/prometheus_cleanup.py @@ -1,11 +1,8 @@ """ Prometheus multiprocess directory cleanup utilities. -When running with multiple workers and PROMETHEUS_MULTIPROC_DIR set, -each worker creates memory-mapped .db files (e.g., counter_1234.db). -When workers die or restart, gauge_live* files for dead PIDs must be -cleaned up via mark_process_dead(). Counter and histogram files are -kept since they contain cumulative data needed for correct aggregation. +mark_process_dead() only removes gauge_live* files — counter/histogram +files are kept for correct aggregation and wiped in bulk on startup. """ from __future__ import annotations @@ -28,39 +25,18 @@ def _get_multiproc_dir() -> Optional[str]: def _is_pid_alive(pid: int) -> bool: - """ - Check if a process with the given PID is alive. - - Uses os.kill(pid, 0) which doesn't send a signal but checks existence. - - ProcessLookupError: process does not exist (dead) - - PermissionError: process exists but we can't signal it (alive, conservative) - - OSError: other error, treat as alive (conservative) - """ + """Check if a process is alive using signal 0 (conservative: unknown = alive).""" try: os.kill(pid, 0) return True except ProcessLookupError: return False - except PermissionError: - # Process exists but we don't have permission to signal it - return True - except OSError: - # Conservative: treat unknown errors as alive + except (PermissionError, OSError): return True def _extract_pids_from_dir(directory: str) -> Set[int]: - """ - Scan .db filenames in a directory and extract PIDs. - - Prometheus client creates files like: - - counter_1234.db - - histogram_1234.db - - gauge_livesum_1234.db - - gauge_liveall_1234.db - - Returns a set of integer PIDs found. - """ + """Scan .db filenames in a directory and extract PIDs (e.g. counter_1234.db -> 1234).""" pids: Set[int] = set() try: for filename in os.listdir(directory): @@ -75,15 +51,7 @@ def _extract_pids_from_dir(directory: str) -> Set[int]: def wipe_directory(directory: str) -> None: - """ - Delete all .db files in the prometheus multiproc directory. - - Called once in the master process before workers fork. Per the - prometheus_client docs: "This directory must be wiped between - process runs (before startup is recommended)." - - Any .db files present at this point are stale from a previous run. - """ + """Delete all .db files in the directory. Called once before workers fork.""" files = glob.glob(os.path.join(directory, "*.db")) for filepath in files: try: @@ -99,14 +67,7 @@ def wipe_directory(directory: str) -> None: def cleanup_own_pid_files() -> None: - """ - Mark the current process as dead for prometheus multiproc cleanup. - - Called during per-worker shutdown. Uses mark_process_dead() which - only removes gauge_live* files — counter and histogram files are - preserved since they contain cumulative data needed for correct - aggregation until the directory is wiped on next startup. - """ + """Mark the current process as dead via mark_process_dead() (worker shutdown).""" directory = _get_multiproc_dir() if not directory or not os.path.isdir(directory): return @@ -126,18 +87,7 @@ def cleanup_own_pid_files() -> None: def mark_dead_pids(skip_pid: Optional[int] = None) -> None: - """ - Scan the prometheus multiproc directory and call mark_process_dead() - for PIDs that no longer exist. - - Uses prometheus_client.multiprocess.mark_process_dead() which only - removes gauge_live* files — counter and histogram files are preserved - since they contain cumulative data needed for correct aggregation. - - Args: - skip_pid: PID to skip (typically os.getpid()). If None, skips - the current process's PID. - """ + """Scan for dead PIDs and call mark_process_dead() for each. Skips current process by default.""" directory = _get_multiproc_dir() if not directory or not os.path.isdir(directory): return From e988867480a54f266613de9a08ed4d96cbf28ae5 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Wed, 25 Feb 2026 17:43:45 -0800 Subject: [PATCH 014/122] wip redundant code --- litellm/proxy/proxy_cli.py | 27 ++++++++++++--------------- litellm/proxy/proxy_server.py | 4 ---- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 1be4931aeef..480f00c3562 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -322,9 +322,6 @@ class ProxyInitializationHelpers: """ Auto-create PROMETHEUS_MULTIPROC_DIR when running with multiple workers and prometheus is configured as a callback. - - If the env var is already set by the user, just ensure the directory exists. - Otherwise, create a temp directory and set the env var. """ import tempfile @@ -341,25 +338,25 @@ class ProxyInitializationHelpers: from litellm.proxy.prometheus_cleanup import wipe_directory - existing_dir = os.environ.get( - "PROMETHEUS_MULTIPROC_DIR" - ) or os.environ.get("prometheus_multiproc_dir") - if existing_dir: - os.makedirs(existing_dir, exist_ok=True) - wipe_directory(existing_dir) - print( # noqa - f"LiteLLM: Using existing PROMETHEUS_MULTIPROC_DIR={existing_dir}" - ) - return + multiproc_dir = ( + os.environ.get("PROMETHEUS_MULTIPROC_DIR") + or os.environ.get("prometheus_multiproc_dir") + ) + auto_created = not multiproc_dir + + if not multiproc_dir: + multiproc_dir = os.path.join( + tempfile.gettempdir(), "litellm_prometheus_multiproc" + ) multiproc_dir = os.path.join( tempfile.gettempdir(), "litellm_prometheus_multiproc" ) os.makedirs(multiproc_dir, exist_ok=True) wipe_directory(multiproc_dir) - os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir + action = "Auto-created" if auto_created else "Use existing" print( # noqa - f"LiteLLM: Auto-created PROMETHEUS_MULTIPROC_DIR={multiproc_dir}" + f"LiteLLM {action}: Auto-created PROMETHEUS_MULTIPROC_DIR={multiproc_dir}" ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c24942e1356..9802d8df6c9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2062,10 +2062,6 @@ def _schedule_background_health_check_db_save( async def _periodic_prometheus_cleanup(): """ Periodically mark dead worker PIDs in the prometheus multiproc directory. - - Uses mark_process_dead() which only removes gauge_live* files, preserving - counter/histogram data for correct aggregation. First run is 1 hour after - startup (startup wipe handles stale files), then every hour thereafter. """ from litellm.proxy.prometheus_cleanup import mark_dead_pids From a0578a29f180c21067e3ff498d8290e6fcbcaee8 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Wed, 25 Feb 2026 17:45:13 -0800 Subject: [PATCH 015/122] fix: dup assignment, os.environ setting --- litellm/proxy/proxy_cli.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 480f00c3562..f5163114983 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -344,20 +344,16 @@ class ProxyInitializationHelpers: ) auto_created = not multiproc_dir - if not multiproc_dir: multiproc_dir = os.path.join( tempfile.gettempdir(), "litellm_prometheus_multiproc" ) - multiproc_dir = os.path.join( - tempfile.gettempdir(), "litellm_prometheus_multiproc" - ) + os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir + os.makedirs(multiproc_dir, exist_ok=True) wipe_directory(multiproc_dir) - action = "Auto-created" if auto_created else "Use existing" - print( # noqa - f"LiteLLM {action}: Auto-created PROMETHEUS_MULTIPROC_DIR={multiproc_dir}" - ) + action = "Auto-created" if auto_created else "Using existing" + print(f"LiteLLM: {action} PROMETHEUS_MULTIPROC_DIR={multiproc_dir}") # noqa @click.command() From 4643685e7838ce0c8fd6c0ed1f12a2a8a34f9e86 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 25 Feb 2026 17:55:26 -0800 Subject: [PATCH 016/122] [Fix] /key/aliases: Add pagination and search to prevent OOMs The /key/aliases endpoint previously fetched all key aliases from the database without limit, causing OOM crashes with large key sets. Added page, size, and search query parameters with database-level filtering to enable paginated and searchable key alias retrieval. Updated the response to include pagination metadata (total_count, current_page, total_pages, size) matching the /v2/model/info pattern. Co-Authored-By: Claude Haiku 4.5 --- .../key_management_endpoints.py | 59 ++++++++++++------- .../test_key_generate_prisma.py | 34 +++++++---- 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c1165ab26d0..8a5145bf793 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4107,13 +4107,23 @@ async def list_keys( dependencies=[Depends(user_api_key_auth)], ) @management_endpoint_wrapper -async def key_aliases() -> Dict[str, List[str]]: +async def key_aliases( + page: int = Query(1, ge=1, description="Page number"), + size: int = Query(50, ge=1, le=100, description="Page size"), + search: Optional[str] = Query( + None, description="Search key aliases (case-insensitive partial match)" + ), +) -> Dict[str, Any]: """ - Lists all key aliases + Lists key aliases with pagination and optional search. Returns: { - "aliases": List[str] + "aliases": List[str], + "total_count": int, + "current_page": int, + "total_pages": int, + "size": int, } """ try: @@ -4125,36 +4135,43 @@ async def key_aliases() -> Dict[str, List[str]]: verbose_proxy_logger.error("Database not connected") raise Exception("Database not connected") - where: Dict[str, Any] = {} + conditions: List[Dict[str, Any]] = [{"key_alias": {"not": None}}] try: - where.update(_get_condition_to_filter_out_ui_session_tokens()) + conditions.append(_get_condition_to_filter_out_ui_session_tokens()) except NameError: # Helper may not exist in some builds; ignore if missing pass + if search: + conditions.append( + {"key_alias": {"contains": search, "mode": "insensitive"}} + ) + where: Dict[str, Any] = {"AND": conditions} + total_count = await prisma_client.db.litellm_verificationtoken.count( + where=where, + ) rows = await prisma_client.db.litellm_verificationtoken.find_many( where=where, order=[{"key_alias": "asc"}], + skip=(page - 1) * size, + take=size, ) - seen = set() - aliases: List[str] = [] - for row in rows: - alias = getattr(row, "key_alias", None) - if alias is None and isinstance(row, dict): - alias = row.get("key_alias") + aliases: List[str] = [row.key_alias for row in rows if row.key_alias] # type: ignore[misc] - if not alias: - continue + total_pages = -(-total_count // size) if total_count > 0 else 0 + verbose_proxy_logger.debug( + f"key_aliases: page={page}, size={size}, search={search!r}, " + f"total_count={total_count}, total_pages={total_pages}" + ) - alias_str = str(alias).strip() - if alias_str and alias_str not in seen: - seen.add(alias_str) - aliases.append(alias_str) - - verbose_proxy_logger.debug(f"Returning {len(aliases)} key aliases") - - return {"aliases": aliases} + return { + "aliases": aliases, + "total_count": total_count, + "current_page": page, + "total_pages": total_pages, + "size": size, + } except Exception as e: verbose_proxy_logger.exception(f"Error in key_aliases: {e}") diff --git a/tests/proxy_unit_tests/test_key_generate_prisma.py b/tests/proxy_unit_tests/test_key_generate_prisma.py index c3f68762810..ed528f21e0d 100644 --- a/tests/proxy_unit_tests/test_key_generate_prisma.py +++ b/tests/proxy_unit_tests/test_key_generate_prisma.py @@ -3668,9 +3668,10 @@ async def test_list_keys(prisma_client): async def test_key_aliases(prisma_client): """ Test the key_aliases function: - - Returns a list + - Returns a paginated response - Includes alias from a newly created key - - Aliases are unique and sorted + - Aliases are sorted + - Pagination and search params work correctly """ import asyncio import uuid @@ -3682,10 +3683,16 @@ async def test_key_aliases(prisma_client): setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") await litellm.proxy.proxy_server.prisma_client.connect() - # Basic call - response = await key_aliases() + # Basic call - check pagination response shape + response = await key_aliases(page=1, size=50) assert "aliases" in response assert isinstance(response["aliases"], list) + assert "total_count" in response + assert "current_page" in response + assert "total_pages" in response + assert "size" in response + assert response["current_page"] == 1 + assert response["size"] == 50 # Create a new user (and key) with a unique alias unique_id = str(uuid.uuid4()) @@ -3704,17 +3711,22 @@ async def test_key_aliases(prisma_client): # Allow async DB writes to settle await asyncio.sleep(2) - # Call again and validate - response_after = await key_aliases() + # Call again and validate alias is present + response_after = await key_aliases(page=1, size=50) aliases = response_after["aliases"] - - # Contains the new alias assert test_alias in aliases - - # Unique & sorted (endpoint dedupes and orders ascending) - assert len(aliases) == len(set(aliases)) assert aliases == sorted(aliases) + # Search by partial alias + partial = test_alias[:10] + search_response = await key_aliases(page=1, size=50, search=partial) + assert test_alias in search_response["aliases"] + + # Search with no match + no_match_response = await key_aliases(page=1, size=50, search="__no_match_xyz__") + assert len(no_match_response["aliases"]) == 0 + assert no_match_response["total_count"] == 0 + @pytest.mark.skip(reason="Requires reliable external DB connection (prisma).") @pytest.mark.asyncio From 0b0809a3d571ed2b1f713dcf5340fe826d9ec76c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 25 Feb 2026 21:03:15 -0800 Subject: [PATCH 017/122] optimize key_aliases to select only key_alias column and add unit tests Add select={"key_alias": True} to the find_many call so only the alias column is fetched from the database instead of full token rows. Add five unit tests in test_key_management_endpoints.py covering response shape, pagination skip/take computation, search filter injection, absence of contains filter when no search term is given, and the select-only-alias optimization. Co-Authored-By: Claude Haiku 4.5 --- .../key_management_endpoints.py | 1 + .../test_key_management_endpoints.py | 98 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 8a5145bf793..8ac6f282719 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4152,6 +4152,7 @@ async def key_aliases( ) rows = await prisma_client.db.litellm_verificationtoken.find_many( where=where, + select={"key_alias": True}, order=[{"key_alias": "asc"}], skip=(page - 1) * size, take=size, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 05df3c2dcbb..8580c89fdaa 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -45,6 +45,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( check_team_key_model_specific_limits, delete_verification_tokens, generate_key_helper_fn, + key_aliases, list_keys, prepare_key_update_data, reset_key_spend_fn, @@ -6211,3 +6212,100 @@ async def test_generate_key_helper_fn_agent_id(): assert key_data.get("agent_id") == "test-agent-456", ( f"Expected agent_id='test-agent-456' in key_data, got: {key_data.get('agent_id')}" ) + + +@pytest.mark.asyncio +async def test_key_aliases_response_shape(): + """Test that key_aliases returns the correct paginated response shape.""" + mock_row1 = MagicMock() + mock_row1.key_alias = "alias-alpha" + mock_row2 = MagicMock() + mock_row2.key_alias = "alias-beta" + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=2) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[mock_row1, mock_row2] + ) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + result = await key_aliases(page=1, size=50, search=None) + + assert result["aliases"] == ["alias-alpha", "alias-beta"] + assert result["total_count"] == 2 + assert result["current_page"] == 1 + assert result["total_pages"] == 1 + assert result["size"] == 50 + + # Both count and find_many must use the same where clause + count_where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] + find_where = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs["where"] + assert count_where == find_where + + # Non-null alias filter must be present + assert json.dumps({"key_alias": {"not": None}}) in json.dumps(count_where) + + +@pytest.mark.asyncio +async def test_key_aliases_pagination_skip_take(): + """Test that skip and take are correctly computed from page and size.""" + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=120) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + result = await key_aliases(page=3, size=25, search=None) + + assert result["current_page"] == 3 + assert result["size"] == 25 + assert result["total_count"] == 120 + assert result["total_pages"] == 5 # ceil(120 / 25) + + find_many_kwargs = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs + assert find_many_kwargs["skip"] == 50 # (3 - 1) * 25 + assert find_many_kwargs["take"] == 25 + + +@pytest.mark.asyncio +async def test_key_aliases_search_filter(): + """Test that the search param adds a case-insensitive contains condition.""" + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + await key_aliases(page=1, size=50, search="my-key") + + where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] + assert ( + json.dumps({"key_alias": {"contains": "my-key", "mode": "insensitive"}}) + in json.dumps(where) + ) + + +@pytest.mark.asyncio +async def test_key_aliases_no_search_omits_contains_filter(): + """Test that without a search term no contains condition is added.""" + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + await key_aliases(page=1, size=50, search=None) + + where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] + assert "contains" not in json.dumps(where) + + +@pytest.mark.asyncio +async def test_key_aliases_select_only_key_alias(): + """Test that find_many is called with select={key_alias: True} to avoid fetching full rows.""" + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + await key_aliases(page=1, size=50, search=None) + + find_many_kwargs = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs + assert find_many_kwargs.get("select") == {"key_alias": True} From 06e87eea877f866af9aa3d32839cac9ffd491595 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 25 Feb 2026 21:40:56 -0800 Subject: [PATCH 018/122] remove unsupported select param from find_many call LiteLLM_VerificationTokenActions.find_many() does not support the select keyword argument. Remove it and drop the corresponding test. Co-Authored-By: Claude Haiku 4.5 --- .../management_endpoints/key_management_endpoints.py | 1 - .../test_key_management_endpoints.py | 12 ------------ 2 files changed, 13 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 8ac6f282719..8a5145bf793 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4152,7 +4152,6 @@ async def key_aliases( ) rows = await prisma_client.db.litellm_verificationtoken.find_many( where=where, - select={"key_alias": True}, order=[{"key_alias": "asc"}], skip=(page - 1) * size, take=size, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 8580c89fdaa..bdfc6734664 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -6297,15 +6297,3 @@ async def test_key_aliases_no_search_omits_contains_filter(): assert "contains" not in json.dumps(where) -@pytest.mark.asyncio -async def test_key_aliases_select_only_key_alias(): - """Test that find_many is called with select={key_alias: True} to avoid fetching full rows.""" - mock_prisma_client = AsyncMock() - mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) - mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) - - with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): - await key_aliases(page=1, size=50, search=None) - - find_many_kwargs = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs - assert find_many_kwargs.get("select") == {"key_alias": True} From d751fdc900887979352a915c8e931c96fef0029d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 11:13:26 +0530 Subject: [PATCH 019/122] add ChatCompletionImageObject in OpenAIChatCompletionAssistantMessage --- litellm/types/llms/openai.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 15e8d1be930..24b18cc488a 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -699,7 +699,15 @@ class OpenAIChatCompletionAssistantMessage(TypedDict, total=False): role: Required[Literal["assistant"]] content: Optional[ Union[ - str, Iterable[Union[ChatCompletionTextObject, ChatCompletionThinkingBlock]] + str, + Iterable[ + Union[ + ChatCompletionTextObject, + ChatCompletionThinkingBlock, + ChatCompletionRedactedThinkingBlock, + ChatCompletionImageObject, + ] + ], ] ] name: Optional[str] From 386c148b8db369a5c2661a17dca1e4065fb661b8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 25 Feb 2026 21:48:53 -0800 Subject: [PATCH 020/122] use raw SQL in key_aliases to avoid loading full rows into memory Replace Prisma ORM count/find_many calls with two query_raw calls that only project the key_alias column. The Prisma client wrapper does not support SELECT projection via find_many, so raw SQL is used to keep memory usage proportional to the page size rather than total key count. Update tests to mock query_raw instead of count/find_many. Co-Authored-By: Claude Haiku 4.5 --- .../key_management_endpoints.py | 52 +++++++----- .../test_key_management_endpoints.py | 79 +++++++++++-------- 2 files changed, 76 insertions(+), 55 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 8a5145bf793..a0442a476f9 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4135,29 +4135,41 @@ async def key_aliases( verbose_proxy_logger.error("Database not connected") raise Exception("Database not connected") - conditions: List[Dict[str, Any]] = [{"key_alias": {"not": None}}] - try: - conditions.append(_get_condition_to_filter_out_ui_session_tokens()) - except NameError: - # Helper may not exist in some builds; ignore if missing - pass + # Build a parameterized WHERE clause to avoid loading full rows into + # memory. Raw SQL is used because the Prisma client wrapper does not + # support column-level SELECT projection on find_many. + # + # $1 is always UI_SESSION_TOKEN_TEAM_ID (filters out UI session tokens). + query_params: List[Any] = [UI_SESSION_TOKEN_TEAM_ID] + where_parts = [ + "key_alias IS NOT NULL", + "key_alias != ''", + "(team_id IS NULL OR team_id != $1)", + ] if search: - conditions.append( - {"key_alias": {"contains": search, "mode": "insensitive"}} - ) - where: Dict[str, Any] = {"AND": conditions} + query_params.append(f"%{search}%") + where_parts.append(f"key_alias ILIKE ${len(query_params)}") - total_count = await prisma_client.db.litellm_verificationtoken.count( - where=where, - ) - rows = await prisma_client.db.litellm_verificationtoken.find_many( - where=where, - order=[{"key_alias": "asc"}], - skip=(page - 1) * size, - take=size, - ) + where_sql = " AND ".join(where_parts) - aliases: List[str] = [row.key_alias for row in rows if row.key_alias] # type: ignore[misc] + count_sql = ( + f'SELECT COUNT(*) AS count FROM "LiteLLM_VerificationToken" WHERE {where_sql}' + ) + count_rows = await prisma_client.db.query_raw(count_sql, *query_params) + total_count = int(count_rows[0]["count"]) if count_rows else 0 + + aliases_params = query_params + [size, (page - 1) * size] + limit_idx = len(aliases_params) - 1 + offset_idx = len(aliases_params) + aliases_sql = ( + f"SELECT key_alias" + f' FROM "LiteLLM_VerificationToken"' + f" WHERE {where_sql}" + f" ORDER BY key_alias ASC" + f" LIMIT ${limit_idx} OFFSET ${offset_idx}" + ) + alias_rows = await prisma_client.db.query_raw(aliases_sql, *aliases_params) + aliases: List[str] = [row["key_alias"] for row in alias_rows if row.get("key_alias")] total_pages = -(-total_count // size) if total_count > 0 else 0 verbose_proxy_logger.debug( diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index bdfc6734664..75325508b01 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -6217,15 +6217,12 @@ async def test_generate_key_helper_fn_agent_id(): @pytest.mark.asyncio async def test_key_aliases_response_shape(): """Test that key_aliases returns the correct paginated response shape.""" - mock_row1 = MagicMock() - mock_row1.key_alias = "alias-alpha" - mock_row2 = MagicMock() - mock_row2.key_alias = "alias-beta" - mock_prisma_client = AsyncMock() - mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=2) - mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[mock_row1, mock_row2] + mock_prisma_client.db.query_raw = AsyncMock( + side_effect=[ + [{"count": 2}], + [{"key_alias": "alias-alpha"}, {"key_alias": "alias-beta"}], + ] ) with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): @@ -6237,21 +6234,23 @@ async def test_key_aliases_response_shape(): assert result["total_pages"] == 1 assert result["size"] == 50 - # Both count and find_many must use the same where clause - count_where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] - find_where = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs["where"] - assert count_where == find_where - - # Non-null alias filter must be present - assert json.dumps({"key_alias": {"not": None}}) in json.dumps(count_where) + # Both SQL calls must filter out null/empty aliases + count_sql = mock_prisma_client.db.query_raw.call_args_list[0].args[0] + aliases_sql = mock_prisma_client.db.query_raw.call_args_list[1].args[0] + assert "key_alias IS NOT NULL" in count_sql + assert "key_alias IS NOT NULL" in aliases_sql @pytest.mark.asyncio async def test_key_aliases_pagination_skip_take(): - """Test that skip and take are correctly computed from page and size.""" + """Test that LIMIT and OFFSET are correctly derived from page and size.""" mock_prisma_client = AsyncMock() - mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=120) - mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.query_raw = AsyncMock( + side_effect=[ + [{"count": 120}], + [], + ] + ) with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): result = await key_aliases(page=3, size=25, search=None) @@ -6261,39 +6260,49 @@ async def test_key_aliases_pagination_skip_take(): assert result["total_count"] == 120 assert result["total_pages"] == 5 # ceil(120 / 25) - find_many_kwargs = mock_prisma_client.db.litellm_verificationtoken.find_many.call_args.kwargs - assert find_many_kwargs["skip"] == 50 # (3 - 1) * 25 - assert find_many_kwargs["take"] == 25 + # aliases query params: [UI_SESSION_TOKEN_TEAM_ID, size=25, offset=50] + aliases_call_args = mock_prisma_client.db.query_raw.call_args_list[1].args + assert aliases_call_args[-2] == 25 # LIMIT = size + assert aliases_call_args[-1] == 50 # OFFSET = (3 - 1) * 25 @pytest.mark.asyncio async def test_key_aliases_search_filter(): - """Test that the search param adds a case-insensitive contains condition.""" + """Test that the search param adds a case-insensitive ILIKE condition.""" mock_prisma_client = AsyncMock() - mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) - mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.query_raw = AsyncMock( + side_effect=[ + [{"count": 0}], + [], + ] + ) with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): await key_aliases(page=1, size=50, search="my-key") - where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] - assert ( - json.dumps({"key_alias": {"contains": "my-key", "mode": "insensitive"}}) - in json.dumps(where) - ) + count_call = mock_prisma_client.db.query_raw.call_args_list[0] + count_sql = count_call.args[0] + count_params = count_call.args[1:] + + assert "ILIKE" in count_sql + assert "%my-key%" in count_params @pytest.mark.asyncio -async def test_key_aliases_no_search_omits_contains_filter(): - """Test that without a search term no contains condition is added.""" +async def test_key_aliases_no_search_omits_ilike_filter(): + """Test that without a search term no ILIKE condition is added.""" mock_prisma_client = AsyncMock() - mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) - mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.query_raw = AsyncMock( + side_effect=[ + [{"count": 0}], + [], + ] + ) with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): await key_aliases(page=1, size=50, search=None) - where = mock_prisma_client.db.litellm_verificationtoken.count.call_args.kwargs["where"] - assert "contains" not in json.dumps(where) + count_sql = mock_prisma_client.db.query_raw.call_args_list[0].args[0] + assert "ILIKE" not in count_sql From f1080a7e3048ab580acb362cbdb6cb1da460f565 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Feb 2026 11:29:34 +0530 Subject: [PATCH 021/122] Add 'image_url; to both if the intent is to support it in assistant messages --- litellm/types/llms/openai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 24b18cc488a..c0aae9bc2de 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -794,17 +794,19 @@ ValidUserMessageContentTypes = [ "file", ] # used for validating user messages. Prevent users from accidentally sending anthropic messages. -# Assistant message content types (text, thinking, redacted_thinking) +# Assistant message content types (text, thinking, redacted_thinking, image_url) ValidAssistantMessageContentTypesLiteral = Literal[ "text", "thinking", "redacted_thinking", + "image_url", ] ValidAssistantMessageContentTypes = [ "text", "thinking", "redacted_thinking", + "image_url", ] # Combined valid content types for chat completion messages From 701ec62da64b4b2e01d03d12b0abf405241f4eb2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 25 Feb 2026 22:21:32 -0800 Subject: [PATCH 022/122] [Feature] UI - Paginated Key Alias Select Replace the non-paginated Key Alias filter with a new PaginatedKeyAliasSelect component that mirrors the existing PaginatedModelSelect pattern. This aligns the UI with the paginated /key/aliases endpoint from PR #22137. Changes: - Added useInfiniteKeyAliases hook for paginated key alias fetching - Created PaginatedKeyAliasSelect component with infinite scroll (80% threshold) - Updated keyAliasesCall in networking to accept page/size/search params - Replaced Key Alias filter in Request Logs and Virtual Keys tables to use customComponent - Removed fetchAllKeyAliases helper and related upfront fetching logic - Added 22 tests for new component and hook; all existing tests pass (54 tests) Fixes the issue where the UI was fetching all key aliases at once, causing performance issues with large key sets. Co-Authored-By: Claude Haiku 4.5 --- ui/litellm-dashboard/package-lock.json | 15 -- .../hooks/keys/useKeyAliases.test.ts | 177 ++++++++++++ .../(dashboard)/hooks/keys/useKeyAliases.ts | 37 +++ .../PaginatedKeyAliasSelect.test.tsx | 255 ++++++++++++++++++ .../PaginatedKeyAliasSelect.tsx | 106 ++++++++ .../VirtualKeysPage/VirtualKeysTable.test.tsx | 9 - .../VirtualKeysPage/VirtualKeysTable.tsx | 17 +- .../key_team_helpers/filter_helpers.ts | 21 +- .../key_team_helpers/filter_logic.tsx | 14 +- .../src/components/networking.tsx | 28 +- .../src/components/view_logs/index.test.tsx | 2 - .../src/components/view_logs/index.tsx | 14 +- .../view_logs/log_filter_logic.test.tsx | 4 +- .../components/view_logs/log_filter_logic.tsx | 13 +- ui/litellm-dashboard/tsconfig.tsbuildinfo | 2 +- 15 files changed, 609 insertions(+), 105 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyAliases.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyAliases.ts create mode 100644 ui/litellm-dashboard/src/components/KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect.test.tsx create mode 100644 ui/litellm-dashboard/src/components/KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect.tsx diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index cc04e674003..fc2aa1599d3 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -13056,21 +13056,6 @@ "type": "github", "url": "https://github.com/sponsors/wooorm" } - }, - "node_modules/@next/swc-win32-ia32-msvc": { - "version": "14.2.33", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.33.tgz", - "integrity": "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } } } } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyAliases.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyAliases.test.ts new file mode 100644 index 00000000000..b382b1f2ad3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyAliases.test.ts @@ -0,0 +1,177 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { renderHook, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import React, { ReactNode } from "react"; +import { useInfiniteKeyAliases } from "./useKeyAliases"; +import type { PaginatedKeyAliasResponse } from "@/components/networking"; + +// Mock networking module +vi.mock("@/components/networking", () => ({ + keyAliasesCall: vi.fn(), +})); + +// Mock useAuthorized hook +const mockUseAuthorized = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +// Mock console methods to avoid noise +vi.spyOn(console, "log").mockImplementation(() => {}); +vi.spyOn(console, "error").mockImplementation(() => {}); + +import { keyAliasesCall } from "@/components/networking"; + +const mockKeyAliasesCall = vi.mocked(keyAliasesCall); + +const mockPage1: PaginatedKeyAliasResponse = { + aliases: ["alias-1", "alias-2"], + total_count: 3, + current_page: 1, + total_pages: 2, + size: 2, +}; + +const mockPage2: PaginatedKeyAliasResponse = { + aliases: ["alias-3"], + total_count: 3, + current_page: 2, + total_pages: 2, + size: 2, +}; + +const createWrapper = () => { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + }, + }); + return ({ children }: { children: ReactNode }) => + React.createElement(QueryClientProvider, { client: queryClient }, children); +}; + +describe("useInfiniteKeyAliases", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockUseAuthorized.mockReturnValue({ accessToken: "test-token" }); + mockKeyAliasesCall.mockResolvedValue(mockPage1); + }); + + it("should fetch the first page of key aliases", async () => { + const wrapper = createWrapper(); + const { result } = renderHook(() => useInfiniteKeyAliases(), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + expect(mockKeyAliasesCall).toHaveBeenCalledWith("test-token", 1, 50, undefined); + expect(result.current.data?.pages[0]).toEqual(mockPage1); + }); + + it("should pass custom size parameter", async () => { + const wrapper = createWrapper(); + renderHook(() => useInfiniteKeyAliases(25), { wrapper }); + + await waitFor(() => { + expect(mockKeyAliasesCall).toHaveBeenCalledWith("test-token", 1, 25, undefined); + }); + }); + + it("should pass search parameter when provided", async () => { + const wrapper = createWrapper(); + renderHook(() => useInfiniteKeyAliases(50, "my-alias"), { wrapper }); + + await waitFor(() => { + expect(mockKeyAliasesCall).toHaveBeenCalledWith("test-token", 1, 50, "my-alias"); + }); + }); + + it("should not fetch when accessToken is not available", () => { + mockUseAuthorized.mockReturnValue({ accessToken: null }); + const wrapper = createWrapper(); + const { result } = renderHook(() => useInfiniteKeyAliases(), { wrapper }); + + expect(result.current.isFetching).toBe(false); + expect(mockKeyAliasesCall).not.toHaveBeenCalled(); + }); + + it("should expose hasNextPage when more pages are available", async () => { + const wrapper = createWrapper(); + const { result } = renderHook(() => useInfiniteKeyAliases(), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.hasNextPage).toBe(true); + }); + + it("should return hasNextPage false when on last page", async () => { + const singlePage: PaginatedKeyAliasResponse = { + aliases: ["alias-1"], + total_count: 1, + current_page: 1, + total_pages: 1, + size: 50, + }; + mockKeyAliasesCall.mockResolvedValue(singlePage); + + const wrapper = createWrapper(); + const { result } = renderHook(() => useInfiniteKeyAliases(), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.hasNextPage).toBe(false); + }); + + it("should fetch the next page when fetchNextPage is called", async () => { + mockKeyAliasesCall + .mockResolvedValueOnce(mockPage1) + .mockResolvedValueOnce(mockPage2); + + const wrapper = createWrapper(); + const { result } = renderHook(() => useInfiniteKeyAliases(2), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + result.current.fetchNextPage(); + + await waitFor(() => { + expect(result.current.data?.pages).toHaveLength(2); + }); + + expect(mockKeyAliasesCall).toHaveBeenCalledWith("test-token", 2, 2, undefined); + expect(result.current.data?.pages[1]).toEqual(mockPage2); + }); + + it("should include search in query key so search changes refetch from page 1", async () => { + const wrapper = createWrapper(); + const { result, rerender } = renderHook( + ({ search }: { search?: string }) => useInfiniteKeyAliases(50, search), + { wrapper, initialProps: { search: undefined } }, + ); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + mockKeyAliasesCall.mockResolvedValue({ + aliases: ["search-result"], + total_count: 1, + current_page: 1, + total_pages: 1, + size: 50, + }); + + rerender({ search: "search-result" }); + + await waitFor(() => { + expect(mockKeyAliasesCall).toHaveBeenCalledWith("test-token", 1, 50, "search-result"); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyAliases.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyAliases.ts new file mode 100644 index 00000000000..f67b15f3a9f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyAliases.ts @@ -0,0 +1,37 @@ +import { useInfiniteQuery } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; +import { keyAliasesCall, type PaginatedKeyAliasResponse } from "@/components/networking"; +import useAuthorized from "../useAuthorized"; + +const infiniteKeyAliasKeys = createQueryKeys("infiniteKeyAliases"); + +export const useInfiniteKeyAliases = ( + size: number = 50, + search?: string, +) => { + const { accessToken } = useAuthorized(); + return useInfiniteQuery({ + queryKey: infiniteKeyAliasKeys.list({ + filters: { + size, + ...(search && { search }), + }, + }), + queryFn: async ({ pageParam }) => { + return await keyAliasesCall( + accessToken!, + pageParam as number, + size, + search, + ); + }, + initialPageParam: 1, + getNextPageParam: (lastPage) => { + if (lastPage.current_page < lastPage.total_pages) { + return lastPage.current_page + 1; + } + return undefined; + }, + enabled: Boolean(accessToken), + }); +}; diff --git a/ui/litellm-dashboard/src/components/KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect.test.tsx b/ui/litellm-dashboard/src/components/KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect.test.tsx new file mode 100644 index 00000000000..9a3755124b7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect.test.tsx @@ -0,0 +1,255 @@ +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders } from "../../../../tests/test-utils"; +import { PaginatedKeyAliasSelect } from "./PaginatedKeyAliasSelect"; + +const mockFetchNextPage = vi.fn(); + +vi.mock("@/app/(dashboard)/hooks/keys/useKeyAliases", () => ({ + useInfiniteKeyAliases: vi.fn(), +})); + +vi.mock("@tanstack/react-pacer/debouncer", async () => { + const React = await vi.importActual("react"); + return { + useDebouncedState: (initial: string) => { + const [value, setValue] = React.useState(initial); + return [value, setValue]; + }, + }; +}); + +import { useInfiniteKeyAliases } from "@/app/(dashboard)/hooks/keys/useKeyAliases"; + +const mockUseInfiniteKeyAliases = vi.mocked(useInfiniteKeyAliases); + +const mockPagesWithAliases = { + pages: [ + { + aliases: ["alias-1", "alias-2"], + total_count: 2, + current_page: 1, + total_pages: 1, + size: 50, + }, + ], +}; + +const mockEmptyPages = { + pages: [{ aliases: [], total_count: 0, current_page: 1, total_pages: 1, size: 50 }], +}; + +describe("PaginatedKeyAliasSelect", () => { + const mockOnChange = vi.fn(); + + const defaultHookReturn = { + data: mockPagesWithAliases, + fetchNextPage: mockFetchNextPage, + hasNextPage: false, + isFetchingNextPage: false, + isLoading: false, + }; + + beforeEach(() => { + vi.clearAllMocks(); + mockUseInfiniteKeyAliases.mockReturnValue(defaultHookReturn as any); + }); + + it("should render", () => { + renderWithProviders(); + + expect(screen.getByRole("combobox")).toBeInTheDocument(); + expect(screen.getByText("Select a key alias")).toBeInTheDocument(); + }); + + it("should display custom placeholder when provided", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("Choose alias")).toBeInTheDocument(); + }); + + it("should display alias options when data is loaded", async () => { + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + expect(screen.getByRole("option", { name: "alias-1" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "alias-2" })).toBeInTheDocument(); + }); + }); + + it("should call onChange when user selects an alias", async () => { + const user = userEvent.setup({ delay: null }); + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await user.click(combobox); + + const option = await screen.findByTitle("alias-1"); + await user.click(option); + + await waitFor(() => { + expect(mockOnChange).toHaveBeenCalledWith("alias-1"); + }); + }); + + it("should show loading state when isLoading is true", () => { + mockUseInfiniteKeyAliases.mockReturnValue({ + ...defaultHookReturn, + isLoading: true, + } as any); + + renderWithProviders(); + + expect(screen.getByRole("combobox")).toHaveAttribute("aria-expanded", "false"); + }); + + it("should pass pageSize to useInfiniteKeyAliases", () => { + renderWithProviders(); + + expect(mockUseInfiniteKeyAliases).toHaveBeenCalledWith(25, undefined); + }); + + it("should pass search to useInfiniteKeyAliases when user types", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await user.click(combobox); + await user.keyboard("my-alias"); + + await waitFor(() => { + expect(mockUseInfiniteKeyAliases).toHaveBeenCalledWith(50, "my-alias"); + }); + }); + + it("should have scroll container for infinite loading when hasNextPage is true", async () => { + mockUseInfiniteKeyAliases.mockReturnValue({ + ...defaultHookReturn, + hasNextPage: true, + isFetchingNextPage: false, + } as any); + + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + expect(screen.getByRole("option", { name: "alias-1" })).toBeInTheDocument(); + }); + + const scrollableContainer = document.querySelector( + ".ant-select-dropdown .rc-virtual-list-holder", + ); + expect(scrollableContainer).toBeInTheDocument(); + }); + + it("should deduplicate aliases with the same value across pages", async () => { + mockUseInfiniteKeyAliases.mockReturnValue({ + ...defaultHookReturn, + data: { + pages: [ + { + aliases: ["alias-1", "alias-1"], + total_count: 2, + current_page: 1, + total_pages: 1, + size: 50, + }, + ], + }, + } as any); + + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + const options = screen.queryAllByRole("option", { name: "alias-1" }); + expect(options.length).toBe(1); + }); + }); + + it("should skip empty aliases", async () => { + mockUseInfiniteKeyAliases.mockReturnValue({ + ...defaultHookReturn, + data: { + pages: [ + { + aliases: ["valid-alias", "", null], + total_count: 3, + current_page: 1, + total_pages: 1, + size: 50, + }, + ], + }, + } as any); + + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + expect(screen.getByRole("option", { name: "valid-alias" })).toBeInTheDocument(); + const allOptions = screen.queryAllByRole("option"); + expect(allOptions.length).toBe(1); + }); + }); + + it("should respect allowClear prop", () => { + renderWithProviders( + , + ); + + expect(screen.getByRole("combobox")).toBeInTheDocument(); + }); + + it("should respect disabled prop", () => { + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + expect(combobox.closest(".ant-select")).toHaveClass("ant-select-disabled"); + }); + + it("should not call fetchNextPage when hasNextPage is false", async () => { + mockUseInfiniteKeyAliases.mockReturnValue({ + ...defaultHookReturn, + hasNextPage: false, + } as any); + + renderWithProviders(); + + await userEvent.click(screen.getByRole("combobox")); + + await waitFor(() => { + expect(screen.getByRole("option", { name: "alias-1" })).toBeInTheDocument(); + }); + + expect(mockFetchNextPage).not.toHaveBeenCalled(); + }); + + it("should show no aliases found when data is empty", async () => { + mockUseInfiniteKeyAliases.mockReturnValue({ + ...defaultHookReturn, + data: mockEmptyPages, + } as any); + + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await userEvent.click(combobox); + + await waitFor(() => { + expect(screen.getByText("No key aliases found")).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect.tsx b/ui/litellm-dashboard/src/components/KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect.tsx new file mode 100644 index 00000000000..0bec77ca52b --- /dev/null +++ b/ui/litellm-dashboard/src/components/KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect.tsx @@ -0,0 +1,106 @@ +import { useInfiniteKeyAliases } from "@/app/(dashboard)/hooks/keys/useKeyAliases"; +import { LoadingOutlined } from "@ant-design/icons"; +import { useDebouncedState } from "@tanstack/react-pacer/debouncer"; +import { Select } from "antd"; +import { useMemo, useState, type UIEvent } from "react"; + +export interface PaginatedKeyAliasSelectProps { + value?: string; + onChange?: (value: string) => void; + placeholder?: string; + style?: React.CSSProperties; + pageSize?: number; + allowClear?: boolean; + disabled?: boolean; +} + +const SCROLL_THRESHOLD = 0.8; +const DEBOUNCE_MS = 300; + +export const PaginatedKeyAliasSelect = ({ + value, + onChange, + placeholder = "Select a key alias", + style, + pageSize = 50, + allowClear = true, + disabled = false, +}: PaginatedKeyAliasSelectProps) => { + const [searchInput, setSearchInput] = useState(""); + const [debouncedSearch, setDebouncedSearch] = useDebouncedState("", { + wait: DEBOUNCE_MS, + }); + + const { + data, + fetchNextPage, + hasNextPage, + isFetchingNextPage, + isLoading, + } = useInfiniteKeyAliases(pageSize, debouncedSearch || undefined); + + const options = useMemo(() => { + if (!data?.pages) return []; + + const seen = new Set(); + const result: { label: string; value: string }[] = []; + + for (const page of data.pages) { + for (const alias of page.aliases) { + if (!alias || seen.has(alias)) continue; + seen.add(alias); + result.push({ label: alias, value: alias }); + } + } + + return result; + }, [data]); + + const handlePopupScroll = (e: UIEvent) => { + const target = e.currentTarget; + const scrollRatio = + (target.scrollTop + target.clientHeight) / target.scrollHeight; + + if (scrollRatio >= SCROLL_THRESHOLD && hasNextPage && !isFetchingNextPage) { + fetchNextPage(); + } + }; + + const handleSearch = (value: string) => { + setSearchInput(value); + setDebouncedSearch(value); + }; + + const handleChange = (v: string | null) => { + onChange?.(v ?? ""); + }; + + return ( + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {/* Advanced Settings */} + + + + + Advanced Settings + + } + key="1" + > + + Block Project + + + + + prev.isBlocked !== cur.isBlocked}> + {({ getFieldValue }) => + getFieldValue("isBlocked") ? ( + + ) : null + } + + + + + + Model-Specific Limits + + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name, ...restField }) => ( + + + + + + + + + + + remove(name)} style={{ color: "#ef4444" }} /> + + ))} + + + + + )} + + + + + + Metadata + + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name, ...restField }) => ( + + + + + + + + remove(name)} style={{ color: "#ef4444" }} /> + + ))} + + + + + )} + + + + + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx new file mode 100644 index 00000000000..40ab5045703 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx @@ -0,0 +1,211 @@ +import { useProjects, ProjectResponse } from "@/app/(dashboard)/hooks/projects/useProjects"; +import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { PlusOutlined } from "@ant-design/icons"; +import { + Button, + Card, + Flex, + Input, + Layout, + Space, + Table, + Tag, + theme, + Tooltip, + Typography, +} from "antd"; +import type { ColumnsType } from "antd/es/table"; +import { LayersIcon, SearchIcon } from "lucide-react"; +import { useEffect, useMemo, useState } from "react"; +import { CreateProjectModal } from "./ProjectModals/CreateProjectModal"; + +const { Title, Text } = Typography; +const { Content } = Layout; + +export function ProjectsPage() { + const { token } = theme.useToken(); + const { data: projects, isLoading } = useProjects(); + const { data: teams } = useTeams(); + + const [isCreateModalVisible, setIsCreateModalVisible] = useState(false); + const [searchText, setSearchText] = useState(""); + const [currentPage, setCurrentPage] = useState(1); + const pageSize = 10; + + useEffect(() => { + setCurrentPage(1); + }, [searchText]); + + // Build a team_id → team_alias lookup from the teams list + const teamAliasMap = useMemo(() => { + const map = new Map(); + for (const team of teams ?? []) { + map.set(team.team_id, team.team_alias ?? team.team_id); + } + return map; + }, [teams]); + + // ---------- filtered data ---------- + const filteredProjects = useMemo(() => { + const list = projects ?? []; + if (!searchText) return list; + const lower = searchText.toLowerCase(); + return list.filter((p) => { + const alias = teamAliasMap.get(p.team_id ?? "") ?? ""; + return ( + (p.project_alias ?? "").toLowerCase().includes(lower) || + p.project_id.toLowerCase().includes(lower) || + (p.description ?? "").toLowerCase().includes(lower) || + alias.toLowerCase().includes(lower) + ); + }); + }, [projects, searchText, teamAliasMap]); + + // ---------- Ant Design columns ---------- + const columns: ColumnsType = [ + { + title: "ID", + dataIndex: "project_id", + key: "project_id", + width: 170, + render: (id: string) => ( + + + {id} + + + ), + }, + { + title: "Name", + dataIndex: "project_alias", + key: "project_alias", + sorter: (a, b) => (a.project_alias ?? "").localeCompare(b.project_alias ?? ""), + render: (alias: string | null) => alias ?? "—", + }, + { + title: "Team", + key: "team", + sorter: (a, b) => { + const aAlias = teamAliasMap.get(a.team_id ?? "") ?? ""; + const bAlias = teamAliasMap.get(b.team_id ?? "") ?? ""; + return aAlias.localeCompare(bAlias); + }, + render: (_: unknown, record: ProjectResponse) => { + const alias = teamAliasMap.get(record.team_id ?? ""); + return alias ?? record.team_id ?? "—"; + }, + }, + { + title: "Models", + key: "models", + render: (_: unknown, record: ProjectResponse) => { + const models = record.models ?? []; + return ( + 0 ? models.join(", ") : "No models"}> + + + + {models.length} + + + + ); + }, + }, + { + title: "Status", + dataIndex: "blocked", + key: "status", + render: (blocked: boolean) => ( + + {blocked ? "Blocked" : "Active"} + + ), + }, + { + title: "Created", + dataIndex: "created_at", + key: "created_at", + sorter: (a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime(), + responsive: ["lg"], + render: (date: string) => new Date(date).toLocaleDateString(), + }, + { + title: "Updated", + dataIndex: "updated_at", + key: "updated_at", + responsive: ["xl"], + render: (date: string) => new Date(date).toLocaleDateString(), + }, + ]; + + return ( + + + + + Projects + + + Manage projects within your teams + + + + + + + + } + placeholder="Search projects by name, ID, description, or team..." + style={{ maxWidth: 400 }} + value={searchText} + onChange={(e) => setSearchText(e.target.value)} + allowClear + /> + + setCurrentPage(page), + size: "small", + showTotal: (total) => `${total} projects`, + showSizeChanger: false, + }} + /> + + + setIsCreateModalVisible(false)} + /> + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/types.ts b/ui/litellm-dashboard/src/components/Projects/types.ts new file mode 100644 index 00000000000..51429902dff --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/types.ts @@ -0,0 +1,14 @@ +export interface Project { + id: string; + name: string; + description: string; + teamId: string; + teamAlias: string; + models: string[]; + status: "active" | "blocked"; + spend: number; + createdAt: string; + createdBy: string; + updatedAt: string; + updatedBy: string; +} diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 2cbeb22ec81..bb0bd54c7a9 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -13,6 +13,7 @@ import { DatabaseOutlined, ExperimentOutlined, FileTextOutlined, + FolderOutlined, KeyOutlined, LineChartOutlined, PlayCircleOutlined, @@ -172,6 +173,23 @@ const menuGroups: MenuGroup[] = [ { groupLabel: "ACCESS CONTROL", items: [ + { + key: "teams", + page: "teams", + label: "Teams", + icon: , + }, + { + key: "projects", + page: "projects", + label: ( + + Projects + + ), + icon: , + roles: all_admin_roles, + }, { key: "users", page: "users", @@ -179,12 +197,6 @@ const menuGroups: MenuGroup[] = [ icon: , roles: all_admin_roles, }, - { - key: "teams", - page: "teams", - label: "Teams", - icon: , - }, { key: "organizations", page: "organizations", @@ -195,11 +207,7 @@ const menuGroups: MenuGroup[] = [ { key: "access-groups", page: "access-groups", - label: ( - - Access Groups - - ), + label: "Access Groups", icon: , roles: all_admin_roles, }, diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 5e2c0ccc0b9..a910373d66d 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -21,6 +21,7 @@ export const pageDescriptions: Record = { users: "Manage internal user accounts and permissions", teams: "Create and manage teams for access control", organizations: "Manage organizations and their members", + projects: "Manage projects within teams", "access-groups": "Manage access groups for role-based permissions", budgets: "Set and monitor spending budgets", api_ref: "Browse API documentation and endpoints", From c47d21980626eded69208a75ec9a36af3c25f464 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 19:04:49 -0300 Subject: [PATCH 089/122] docs: add openrouter/anthropic/claude-opus-4.6 to main model cost map --- model_prices_and_context_window.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 66c8e55c3ae..186306b5b56 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23510,6 +23510,25 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "openrouter/anthropic/claude-opus-4.6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 3.75e-06, From 5628ba0831d3c0f0d4c425f2a286a1debb36ae61 Mon Sep 17 00:00:00 2001 From: Chesars Date: Fri, 27 Feb 2026 19:05:33 -0300 Subject: [PATCH 090/122] docs: remove openrouter/anthropic/claude-opus-4.6 from backup JSON Model should only be in the main cost map, not the backup. --- ...odel_prices_and_context_window_backup.json | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 186306b5b56..66c8e55c3ae 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23510,25 +23510,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-opus-4.6": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 346 - }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 3.75e-06, From 8bb64574712e907186d0a76d9f26f17ffee7567f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 27 Feb 2026 15:41:03 -0800 Subject: [PATCH 091/122] [Fix] Include created_at and updated_at in /project/list response The /project/list endpoint was not returning created_at and updated_at timestamps because these fields were not defined in LiteLLM_ProjectTable. Added these fields to the model so FastAPI includes them in the response (values come from the database). This allows the UI to display project creation and last-updated times. Co-Authored-By: Claude Haiku 4.5 --- litellm/proxy/_types.py | 2 + .../test_project_endpoints_prisma.py | 75 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 28311ab1b3b..dfc2ba59d96 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2665,6 +2665,8 @@ class LiteLLM_ProjectTable(LiteLLMPydanticObjectBase): object_permission_id: Optional[str] = None created_by: str updated_by: str + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None litellm_budget_table: Optional[LiteLLM_BudgetTable] = None object_permission: Optional[LiteLLM_ObjectPermissionTable] = None diff --git a/tests/proxy_unit_tests/test_project_endpoints_prisma.py b/tests/proxy_unit_tests/test_project_endpoints_prisma.py index c98cb7efda0..77ed09a40f0 100644 --- a/tests/proxy_unit_tests/test_project_endpoints_prisma.py +++ b/tests/proxy_unit_tests/test_project_endpoints_prisma.py @@ -791,3 +791,78 @@ def test_litellm_entity_type_has_project(): assert hasattr(Litellm_EntityType, "PROJECT") assert Litellm_EntityType.PROJECT.value == "project" + + +@pytest.mark.asyncio +async def test_list_projects_returns_timestamps(): + """ + Test that /project/list returns created_at and updated_at for each project. + """ + from datetime import datetime, timezone + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy.management_endpoints.project_endpoints import list_projects + from litellm.proxy._types import LiteLLM_ProjectTable + + now = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc) + + # Build a fake DB row that includes created_at and updated_at + fake_project = MagicMock() + fake_project.model_dump.return_value = { + "project_id": "proj-1", + "project_alias": "test-project", + "team_id": "team-1", + "created_by": "admin", + "updated_by": "admin", + "created_at": now, + "updated_at": now, + "models": [], + "spend": 0.0, + "blocked": False, + "budget_id": None, + "description": None, + "metadata": None, + "model_spend": None, + "model_rpm_limit": None, + "model_tpm_limit": None, + "object_permission_id": None, + "litellm_budget_table": None, + "object_permission": None, + } + # Make the fake row behave like a Pydantic model for FastAPI serialization + fake_project.project_id = "proj-1" + fake_project.created_at = now + fake_project.updated_at = now + + mock_prisma = MagicMock() + mock_prisma.db.litellm_projecttable.find_many = AsyncMock( + return_value=[fake_project] + ) + + with patch( + "litellm.proxy.proxy_server.prisma_client", mock_prisma + ): + response = await list_projects( + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + ) + + assert len(response) == 1 + project = response[0] + assert project.created_at == now + assert project.updated_at == now + + +def test_litellm_project_table_has_timestamp_fields(): + """ + Test that LiteLLM_ProjectTable model includes created_at and updated_at fields, + so the /project/list response_model exposes them. + """ + from litellm.proxy._types import LiteLLM_ProjectTable + + fields = LiteLLM_ProjectTable.model_fields + assert "created_at" in fields, "LiteLLM_ProjectTable must have created_at field" + assert "updated_at" in fields, "LiteLLM_ProjectTable must have updated_at field" From 64c85dbc9fd3673b76693bbedf604ed6ff25d5d4 Mon Sep 17 00:00:00 2001 From: Rahul Dhanawade Date: Sat, 28 Feb 2026 05:29:37 +0530 Subject: [PATCH 092/122] Fix/claude code plugin schema (#22271) * fix: add missing LiteLLM_ClaudeCodePluginTable to schema.prisma - Claude Code Plugin Marketplace endpoints (/claude-code/marketplace.json, /claude-code/plugins) were returning 500 errors because LiteLLM_ClaudeCodePluginTable model was missing from both schema.prisma files - Prisma client was generated without this table causing AttributeError: 'Prisma' object has no attribute 'litellm_claudecodeplugintable' - Added missing model definition to root schema.prisma and litellm/proxy/schema.prisma Fixes #21310 * test: add regression test for LiteLLM_ClaudeCodePluginTable schema * fix: address greptile review - add @updatedAt, clean up test imports --- litellm/proxy/schema.prisma | 17 ++++++++++++++++- schema.prisma | 17 ++++++++++++++++- .../proxy/test_claude_code_marketplace.py | 18 ++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/litellm/proxy/test_claude_code_marketplace.py diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 13461be3e7c..a5b0d930f58 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1096,4 +1096,19 @@ model LiteLLM_AccessGroupTable { created_by String? updated_at DateTime @default(now()) @updatedAt updated_by String? -} \ No newline at end of file +} +// Claude Code Plugin Marketplace table +model LiteLLM_ClaudeCodePluginTable { + id String @id @default(uuid()) + name String @unique + version String? + description String? + manifest_json String? + files_json String? @default("{}") + enabled Boolean @default(true) + created_at DateTime? @default(now()) + updated_at DateTime? @default(now()) @updatedAt + created_by String? + + @@map("LiteLLM_ClaudeCodePluginTable") +} diff --git a/schema.prisma b/schema.prisma index 34308b29ebf..bc32a8cce32 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1095,4 +1095,19 @@ model LiteLLM_AccessGroupTable { created_by String? updated_at DateTime @default(now()) @updatedAt updated_by String? -} \ No newline at end of file +} +// Claude Code Plugin Marketplace table +model LiteLLM_ClaudeCodePluginTable { + id String @id @default(uuid()) + name String @unique + version String? + description String? + manifest_json String? + files_json String? @default("{}") + enabled Boolean @default(true) + created_at DateTime? @default(now()) + updated_at DateTime? @default(now()) @updatedAt + created_by String? + + @@map("LiteLLM_ClaudeCodePluginTable") +} diff --git a/tests/litellm/proxy/test_claude_code_marketplace.py b/tests/litellm/proxy/test_claude_code_marketplace.py new file mode 100644 index 00000000000..5376e81012b --- /dev/null +++ b/tests/litellm/proxy/test_claude_code_marketplace.py @@ -0,0 +1,18 @@ +import pytest + + +@pytest.mark.asyncio +async def test_claude_code_plugin_table_schema_exists(): + + with open("schema.prisma", "r") as f: + schema = f.read() + assert "LiteLLM_ClaudeCodePluginTable" in schema, ( + "LiteLLM_ClaudeCodePluginTable model missing from schema.prisma - " + "this causes AttributeError on all /claude-code/plugins endpoints" + ) + + with open("litellm/proxy/schema.prisma", "r") as f: + proxy_schema = f.read() + assert "LiteLLM_ClaudeCodePluginTable" in proxy_schema, ( + "LiteLLM_ClaudeCodePluginTable model missing from litellm/proxy/schema.prisma" + ) From 4fa6742b01e60cd9b4d9c44e4ebc893947edfc9e Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Fri, 27 Feb 2026 16:07:29 -0800 Subject: [PATCH 093/122] Add Prometheus child_exit cleanup for gunicorn workers When a gunicorn worker exits (e.g. from max_requests recycling), its per-process prometheus .db files remain on disk. For gauges using livesum/liveall mode, this means the dead worker's last-known values persist as if the process were still alive. Wire gunicorn's child_exit hook to call mark_process_dead() so live-tracking gauges accurately reflect only running workers. --- litellm/proxy/prometheus_cleanup.py | 17 ++++++++++ litellm/proxy/proxy_cli.py | 9 ++++++ .../proxy/test_prometheus_cleanup.py | 31 ++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/prometheus_cleanup.py b/litellm/proxy/prometheus_cleanup.py index 6d935a8dd90..6353588532a 100644 --- a/litellm/proxy/prometheus_cleanup.py +++ b/litellm/proxy/prometheus_cleanup.py @@ -28,3 +28,20 @@ def wipe_directory(directory: str) -> None: verbose_proxy_logger.info( f"Prometheus cleanup: wiped {deleted} stale .db files from {directory}" ) + + +def mark_worker_exit(worker_pid: int) -> None: + """Remove prometheus .db files for a dead worker. Called by gunicorn child_exit hook.""" + if not os.environ.get("PROMETHEUS_MULTIPROC_DIR"): + return + try: + from prometheus_client import multiprocess + + multiprocess.mark_process_dead(worker_pid) + verbose_proxy_logger.info( + f"Prometheus cleanup: marked worker {worker_pid} as dead" + ) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to mark prometheus worker {worker_pid} as dead: {e}" + ) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index f5163114983..921d86c35c1 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -277,6 +277,15 @@ class ProxyInitializationHelpers: if max_requests_before_restart is not None: gunicorn_options["max_requests"] = max_requests_before_restart + # Clean up prometheus .db files when a worker exits (prevents ghost gauge values) + if os.environ.get("PROMETHEUS_MULTIPROC_DIR"): + from litellm.proxy.prometheus_cleanup import mark_worker_exit + + def child_exit(server, worker): + mark_worker_exit(worker.pid) + + gunicorn_options["child_exit"] = child_exit + if ssl_certfile_path is not None and ssl_keyfile_path is not None: print( # noqa f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n" # noqa diff --git a/tests/test_litellm/proxy/test_prometheus_cleanup.py b/tests/test_litellm/proxy/test_prometheus_cleanup.py index 276f2b592db..b3d785f1133 100644 --- a/tests/test_litellm/proxy/test_prometheus_cleanup.py +++ b/tests/test_litellm/proxy/test_prometheus_cleanup.py @@ -10,7 +10,7 @@ from unittest.mock import patch import pytest -from litellm.proxy.prometheus_cleanup import wipe_directory +from litellm.proxy.prometheus_cleanup import mark_worker_exit, wipe_directory from litellm.proxy.proxy_cli import ProxyInitializationHelpers @@ -23,6 +23,35 @@ class TestWipeDirectory: assert not list(tmp_path.glob("*.db")) +class TestMarkWorkerExit: + def test_calls_mark_process_dead_when_env_set(self, tmp_path): + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark: + mark_worker_exit(12345) + mock_mark.assert_called_once_with(12345) + + def test_noop_when_env_not_set(self): + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + with patch( + "prometheus_client.multiprocess.mark_process_dead" + ) as mock_mark: + mark_worker_exit(12345) + mock_mark.assert_not_called() + + def test_exception_is_caught_and_logged(self, tmp_path): + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": str(tmp_path)}): + with patch( + "prometheus_client.multiprocess.mark_process_dead", + side_effect=FileNotFoundError("gone"), + ) as mock_mark: + # Should not raise + mark_worker_exit(99) + mock_mark.assert_called_once_with(99) + + class TestMaybeSetupPrometheusMultiprocDir: def test_respects_existing_env_var(self, tmp_path): """When PROMETHEUS_MULTIPROC_DIR is already set, don't override it.""" From af6fe184fb14087b3b9dc7b08628695d21b7ac92 Mon Sep 17 00:00:00 2001 From: Dylan Duan Date: Fri, 27 Feb 2026 20:24:48 -0500 Subject: [PATCH 094/122] docs: update AssemblyAI docs with Universal-3 Pro, Speech Understanding, and LLM Gateway (#21130) * docs: update AssemblyAI docs with Universal-3 Pro, Speech Understanding, and LLM Gateway provider config * feat: add AssemblyAI LLM Gateway as OpenAI-compatible provider --- .../docs/pass_through/assembly_ai.md | 179 ++++++++++++++---- litellm/llms/openai_like/providers.json | 4 + .../openai_like/test_assemblyai_provider.py | 77 ++++++++ 3 files changed, 225 insertions(+), 35 deletions(-) create mode 100644 tests/litellm/llms/openai_like/test_assemblyai_provider.py diff --git a/docs/my-website/docs/pass_through/assembly_ai.md b/docs/my-website/docs/pass_through/assembly_ai.md index 4606640c5c4..c7c70639e7e 100644 --- a/docs/my-website/docs/pass_through/assembly_ai.md +++ b/docs/my-website/docs/pass_through/assembly_ai.md @@ -1,31 +1,36 @@ -# Assembly AI +# AssemblyAI -Pass-through endpoints for Assembly AI - call Assembly AI endpoints, in native format (no translation). +Pass-through endpoints for AssemblyAI - call AssemblyAI endpoints, in native format (no translation). -| Feature | Supported | Notes | +| Feature | Supported | Notes | |-------|-------|-------| | Cost Tracking | ✅ | works across all integrations | | Logging | ✅ | works across all integrations | -Supports **ALL** Assembly AI Endpoints +Supports **ALL** AssemblyAI Endpoints -[**See All Assembly AI Endpoints**](https://www.assemblyai.com/docs/api-reference) +[**See All AssemblyAI Endpoints**](https://www.assemblyai.com/docs/api-reference) - +## Supported Routes + +| AssemblyAI Service | LiteLLM Route | AssemblyAI Base URL | +|-------------------|---------------|---------------------| +| Speech-to-Text (US) | `/assemblyai/*` | `api.assemblyai.com` | +| Speech-to-Text (EU) | `/eu.assemblyai/*` | `eu.api.assemblyai.com` | ## Quick Start -Let's call the Assembly AI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts) +Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts) -1. Add Assembly AI API Key to your environment +1. Add AssemblyAI API Key to your environment ```bash export ASSEMBLYAI_API_KEY="" ``` -2. Start LiteLLM Proxy +2. Start LiteLLM Proxy ```bash litellm @@ -33,53 +38,157 @@ litellm # RUNNING on http://0.0.0.0:4000 ``` -3. Test it! +3. Test it! -Let's call the Assembly AI `/v2/transcripts` endpoint +Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts). Includes commented-out [Speech Understanding](https://www.assemblyai.com/docs/speech-understanding) features you can toggle on. ```python import assemblyai as aai -LITELLM_VIRTUAL_KEY = "sk-1234" # -LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/assemblyai" # /assemblyai +aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # /assemblyai +aai.settings.api_key = "Bearer sk-1234" # Bearer -aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}" -aai.settings.base_url = LITELLM_PROXY_BASE_URL +# Use a publicly-accessible URL +audio_file = "https://assembly.ai/wildfires.mp3" -# URL of the file to transcribe -FILE_URL = "https://assembly.ai/wildfires.mp3" +# Or use a local file: +# audio_file = "./example.mp3" -# You can also transcribe a local file by passing in a file path -# FILE_URL = './path/to/file.mp3' +config = aai.TranscriptionConfig( + speech_models=["universal-3-pro", "universal-2"], + language_detection=True, + speaker_labels=True, + # Speech understanding features + # sentiment_analysis=True, + # entity_detection=True, + # auto_chapters=True, + # summarization=True, + # summary_type=aai.SummarizationType.bullets, + # redact_pii=True, + # content_safety=True, +) -transcriber = aai.Transcriber() -transcript = transcriber.transcribe(FILE_URL) -print(transcript) -print(transcript.id) +transcript = aai.Transcriber().transcribe(audio_file, config=config) + +if transcript.status == aai.TranscriptStatus.error: + raise RuntimeError(f"Transcription failed: {transcript.error}") + +print(f"\nFull Transcript:\n\n{transcript.text}") + +# Optionally print speaker diarization results +# for utterance in transcript.utterances: +# print(f"Speaker {utterance.speaker}: {utterance.text}") ``` -## Calling Assembly AI EU endpoints +4. [Prompting with Universal-3 Pro](https://www.assemblyai.com/docs/speech-to-text/prompting) (optional) -If you want to send your request to the Assembly AI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `/eu.assemblyai` +```python +import assemblyai as aai + +aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # /assemblyai +aai.settings.api_key = "Bearer sk-1234" # Bearer + +audio_file = "https://assemblyaiassets.com/audios/verbatim.mp3" + +config = aai.TranscriptionConfig( + speech_models=["universal-3-pro", "universal-2"], + language_detection=True, + prompt="Produce a transcript suitable for conversational analysis. Every disfluency is meaningful data. Include: fillers (um, uh, er, ah, hmm, mhm, like, you know, I mean), repetitions (I I, the the), restarts (I was- I went), stutters (th-that, b-but, no-not), and informal speech (gonna, wanna, gotta)", +) + +transcript = aai.Transcriber().transcribe(audio_file, config) + +print(transcript.text) +``` + +## Calling AssemblyAI EU endpoints + +If you want to send your request to the AssemblyAI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `/eu.assemblyai` ```python import assemblyai as aai -LITELLM_VIRTUAL_KEY = "sk-1234" # -LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/eu.assemblyai" # /eu.assemblyai +aai.settings.base_url = "http://0.0.0.0:4000/eu.assemblyai" # /eu.assemblyai +aai.settings.api_key = "Bearer sk-1234" # Bearer -aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}" -aai.settings.base_url = LITELLM_PROXY_BASE_URL +# Use a publicly-accessible URL +audio_file = "https://assembly.ai/wildfires.mp3" -# URL of the file to transcribe -FILE_URL = "https://assembly.ai/wildfires.mp3" - -# You can also transcribe a local file by passing in a file path -# FILE_URL = './path/to/file.mp3' +# Or use a local file: +# audio_file = "./path/to/file.mp3" transcriber = aai.Transcriber() -transcript = transcriber.transcribe(FILE_URL) +transcript = transcriber.transcribe(audio_file) print(transcript) print(transcript.id) ``` + +## LLM Gateway + +Use AssemblyAI's [LLM Gateway](https://www.assemblyai.com/docs/llm-gateway) as an OpenAI-compatible provider — a unified API for Claude, GPT, and Gemini models with full LiteLLM logging, guardrails, and cost tracking support. + +[**See Available Models**](https://www.assemblyai.com/docs/llm-gateway#available-models) + +### Usage + +#### LiteLLM Python SDK + +```python +import litellm +import os + +os.environ["ASSEMBLYAI_API_KEY"] = "your-assemblyai-api-key" + +response = litellm.completion( + model="assemblyai/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "What is the capital of France?"}] +) + +print(response.choices[0].message.content) +``` + +#### LiteLLM Proxy + +1. Config + +```yaml +model_list: + - model_name: assemblyai/* + litellm_params: + model: assemblyai/* + api_key: os.environ/ASSEMBLYAI_API_KEY +``` + +2. Start proxy + +```bash +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +3. Test it! + +```python +import requests + +headers = { + "authorization": "Bearer sk-1234" # Bearer +} + +response = requests.post( + "http://0.0.0.0:4000/v1/chat/completions", + headers=headers, + json={ + "model": "assemblyai/claude-sonnet-4-5-20250929", + "messages": [ + {"role": "user", "content": "What is the capital of France?"} + ], + "max_tokens": 1000 + } +) + +result = response.json() +print(result["choices"][0]["message"]["content"]) +``` diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 1b1b1c2f8cc..b3125d4ad38 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -90,5 +90,9 @@ "headers": { "api-subscription-key": "{api_key}" } + }, + "assemblyai": { + "base_url": "https://llm-gateway.assemblyai.com/v1", + "api_key_env": "ASSEMBLYAI_API_KEY" } } diff --git a/tests/litellm/llms/openai_like/test_assemblyai_provider.py b/tests/litellm/llms/openai_like/test_assemblyai_provider.py new file mode 100644 index 00000000000..7eee810b271 --- /dev/null +++ b/tests/litellm/llms/openai_like/test_assemblyai_provider.py @@ -0,0 +1,77 @@ +""" +Unit tests for the AssemblyAI LLM Gateway OpenAI-like provider. +""" + +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +) + +from litellm.llms.openai_like.dynamic_config import create_config_class +from litellm.llms.openai_like.json_loader import JSONProviderRegistry + +ASSEMBLYAI_BASE_URL = "https://llm-gateway.assemblyai.com/v1" + + +def _get_config(): + provider = JSONProviderRegistry.get("assemblyai") + assert provider is not None + config_class = create_config_class(provider) + return config_class() + + +def test_assemblyai_provider_registered(): + provider = JSONProviderRegistry.get("assemblyai") + assert provider is not None + assert provider.base_url == ASSEMBLYAI_BASE_URL + assert provider.api_key_env == "ASSEMBLYAI_API_KEY" + + +def test_assemblyai_resolves_env_api_key(monkeypatch): + config = _get_config() + monkeypatch.setenv("ASSEMBLYAI_API_KEY", "test-key") + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == ASSEMBLYAI_BASE_URL + assert api_key == "test-key" + + +def test_assemblyai_complete_url_appends_endpoint(): + config = _get_config() + url = config.get_complete_url( + api_base=ASSEMBLYAI_BASE_URL, + api_key="test-key", + model="assemblyai/claude-sonnet-4-5-20250929", + optional_params={}, + litellm_params={}, + stream=False, + ) + assert url == f"{ASSEMBLYAI_BASE_URL}/chat/completions" + + +def test_assemblyai_provider_resolution(): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="assemblyai/claude-sonnet-4-5-20250929", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "claude-sonnet-4-5-20250929" + assert provider == "assemblyai" + assert api_base == ASSEMBLYAI_BASE_URL + + +def test_assemblyai_provider_config_manager(): + from litellm import LlmProviders + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_chat_config( + model="claude-sonnet-4-5-20250929", provider=LlmProviders.ASSEMBLYAI + ) + + assert config is not None + assert config.custom_llm_provider == "assemblyai" From cb4cfa1db44c34163d299b1ad2433bab88476ef5 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 27 Feb 2026 22:37:05 -0300 Subject: [PATCH 095/122] fix(mcp): update test mocks to use renamed filter_server_ids_by_ip_with_info Tests were mocking the old method name `filter_server_ids_by_ip` but production code at server.py:774 calls `filter_server_ids_by_ip_with_info` which returns a (server_ids, blocked_count) tuple. The unmocked method on AsyncMock returned a coroutine, causing "cannot unpack non-iterable coroutine object" errors. Co-Authored-By: Claude Opus 4.6 --- tests/mcp_tests/test_mcp_server.py | 66 +++++++++++++++--------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index dc1e2068365..a81702d0db3 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -659,9 +659,9 @@ async def test_list_tools_rest_api_server_not_found(): mock_manager.get_allowed_mcp_servers = AsyncMock( return_value=["non_existent_server_id"] ) - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) # Return None when trying to get the server (server doesn't exist) mock_manager.get_mcp_server_by_id = MagicMock(return_value=None) @@ -732,9 +732,9 @@ async def test_list_tools_rest_api_success(): return_value=["test-server-123"] ) mock_manager.get_mcp_server_by_id = MagicMock(return_value=mock_server) - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) # Mock the _get_tools_for_single_server function @@ -814,9 +814,9 @@ async def test_get_tools_from_mcp_servers(): ) mock_manager.get_mcp_server_by_id = lambda server_id: mock_server_1 if server_id == "server1_id" else mock_server_2 mock_manager._get_tools_from_server = AsyncMock(return_value=[mock_tool_1]) - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) with patch( @@ -853,9 +853,9 @@ async def test_get_tools_from_mcp_servers(): mock_manager_2._get_tools_from_server = AsyncMock( side_effect=mock_get_tools_side_effect ) - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager_2.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager_2.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) with patch( @@ -881,9 +881,9 @@ async def test_get_tools_from_mcp_servers(): ) mock_manager.get_mcp_server_by_id = lambda server_id: mock_server_1 if server_id == "server1_id" else (mock_server_2 if server_id == "server2_id" else mock_server_3) mock_manager._get_tools_from_server = AsyncMock(return_value=[mock_tool_1]) - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) with patch( @@ -1817,9 +1817,9 @@ async def test_list_tool_rest_api_with_server_specific_auth(): mock_server.mcp_info = {"server_name": "zapier"} mock_manager.get_mcp_server_by_id.return_value = mock_server - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) mock_user_api_key_dict = UserAPIKeyAuth( @@ -1911,9 +1911,9 @@ async def test_list_tool_rest_api_with_default_auth(): mock_server.mcp_info = {"server_name": "unknown_server"} mock_manager.get_mcp_server_by_id.return_value = mock_server - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) mock_user_api_key_dict = UserAPIKeyAuth( @@ -2021,9 +2021,9 @@ async def test_list_tool_rest_api_all_servers_with_auth(): server_id ) ) - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) mock_user_api_key_dict = UserAPIKeyAuth( @@ -2154,9 +2154,9 @@ async def test_filter_tools_by_allowed_tools_integration(): return_value=["test-server-123"] ) mock_manager.get_mcp_server_by_id = MagicMock(return_value=mock_server) - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) # Mock the _get_tools_from_server method to return all tools @@ -2268,9 +2268,9 @@ async def test_filter_tools_by_disallowed_tools_integration(): return_value=["test-server-456"] ) mock_manager.get_mcp_server_by_id = MagicMock(return_value=mock_server) - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) # Mock the _get_tools_from_server method to return all tools mock_manager._get_tools_from_server = AsyncMock(return_value=mock_tools) @@ -2368,9 +2368,9 @@ async def test_filter_tools_no_restrictions_integration(): return_value=["test-server-000"] ) mock_manager.get_mcp_server_by_id = MagicMock(return_value=mock_server) - # Mock filter_server_ids_by_ip to return input unchanged (no IP filtering in test) - mock_manager.filter_server_ids_by_ip = MagicMock( - side_effect=lambda server_ids, client_ip: server_ids + # Mock filter_server_ids_by_ip_with_info to return input unchanged (no IP filtering in test) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock( + side_effect=lambda server_ids, client_ip: (server_ids, 0) ) # Mock the _get_tools_from_server method to return all tools From 9a48c8e36ac3c724769f6a3d101ef6b7f0f41764 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 27 Feb 2026 22:53:09 -0300 Subject: [PATCH 096/122] fix(test): update realtime guardrail test assertions for voice violation behavior Tests were asserting no response.create/conversation.item.create sent to backend when guardrail blocks, but the implementation intentionally sends these to have the LLM voice the guardrail violation message to the user. Updated assertions to verify the correct guardrail flow: - response.cancel is sent to stop any in-progress response - conversation.item.create with violation message is injected - response.create is sent to voice the violation - original blocked content is NOT forwarded Co-Authored-By: Claude Opus 4.6 --- .../test_realtime_streaming.py | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 11d6bb028d8..56c1cfa8515 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -430,19 +430,36 @@ async def test_realtime_guardrail_blocks_prompt_injection(): streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) await streaming.backend_to_client_send_messages() - # ASSERT 1: no response.create was sent to backend (injection blocked). + # ASSERT 1: the guardrail blocked the normal auto-response and instead + # injected a conversation.item.create + response.create to voice the + # violation message. There should be exactly ONE response.create (the + # guardrail-triggered one), preceded by a response.cancel and a + # conversation.item.create carrying the violation text. sent_to_backend = [ json.loads(c.args[0]) for c in backend_ws.send.call_args_list if c.args ] - response_creates = [ - e for e in sent_to_backend - if e.get("type") == "response.create" + response_cancels = [ + e for e in sent_to_backend if e.get("type") == "response.cancel" ] - assert len(response_creates) == 0, ( - f"Guardrail should prevent response.create for injected content, " - f"but got: {response_creates}" + assert len(response_cancels) == 1, ( + f"Guardrail should send response.cancel, got: {response_cancels}" + ) + guardrail_items = [ + e for e in sent_to_backend + if e.get("type") == "conversation.item.create" + ] + assert len(guardrail_items) == 1, ( + f"Guardrail should inject a conversation.item.create with violation message, " + f"got: {guardrail_items}" + ) + response_creates = [ + e for e in sent_to_backend if e.get("type") == "response.create" + ] + assert len(response_creates) == 1, ( + f"Guardrail should send exactly one response.create to voice the violation, " + f"got: {response_creates}" ) # ASSERT 2: error event was sent directly to the client WebSocket @@ -595,14 +612,26 @@ async def test_realtime_text_input_guardrail_blocks_and_returns_error(): assert len(error_events) == 1, f"Expected one error event, got: {sent_texts}" assert error_events[0]["error"]["type"] == "guardrail_violation" - # ASSERT: blocked item was NOT forwarded to the backend + # ASSERT: the original blocked item was NOT forwarded to the backend. + # The guardrail handler injects its own conversation.item.create with + # the violation message — only that one should be present, not the + # original user message. sent_to_backend = [c.args[0] for c in backend_ws.send.call_args_list if c.args] forwarded_items = [ json.loads(m) for m in sent_to_backend if isinstance(m, str) and json.loads(m).get("type") == "conversation.item.create" ] - assert len(forwarded_items) == 0, ( - f"Blocked item should not be forwarded to backend, got: {forwarded_items}" + # Filter out guardrail-injected items (contain "Say exactly the following message") + original_items = [ + item for item in forwarded_items + if not any( + "Say exactly the following message" in c.get("text", "") + for c in item.get("item", {}).get("content", []) + if isinstance(c, dict) + ) + ] + assert len(original_items) == 0, ( + f"Blocked item should not be forwarded to backend, got: {original_items}" ) litellm.callbacks = [] # cleanup From aa899b504f008bea404ab46180d38304cf2d6e21 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 27 Feb 2026 22:56:38 -0300 Subject: [PATCH 097/122] fix(bedrock): restore parallel_tool_calls mapping in map_openai_params The revert in 8565c70e53 removed the parallel_tool_calls handling from map_openai_params, and the subsequent fix d0445e1e33 only re-added the transform_request consumption but forgot to re-add the map_openai_params producer that sets _parallel_tool_use_config. This meant parallel_tool_calls was silently ignored for all Bedrock models. Co-Authored-By: Claude Opus 4.6 --- litellm/llms/bedrock/chat/converse_transformation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 306d63b77d0..a0f2f65fb7f 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -511,6 +511,7 @@ class AmazonConverseConfig(BaseConfig): "response_format", "requestMetadata", "service_tier", + "parallel_tool_calls", ] if ( @@ -913,6 +914,13 @@ class AmazonConverseConfig(BaseConfig): ) if _tool_choice_value is not None: optional_params["tool_choice"] = _tool_choice_value + if param == "parallel_tool_calls": + disable_parallel = not value + optional_params["_parallel_tool_use_config"] = { + "tool_choice": { + "disable_parallel_tool_use": disable_parallel + } + } if param == "thinking": optional_params["thinking"] = value elif param == "reasoning_effort" and isinstance(value, str): From 273994b996d1110816d8a8a5e0e1bda3829b467c Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 27 Feb 2026 22:59:38 -0300 Subject: [PATCH 098/122] fix(test): update Azure pass-through test to mock litellm.completion Commit 99c62ca40e removed "azure" from _RESPONSES_API_PROVIDERS, routing Azure models through litellm.completion instead of litellm.responses. The test was not updated to match, causing it to assert against the wrong mock. Co-Authored-By: Claude Opus 4.6 --- ...c_experimental_pass_through_messages_handler.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index c671d9b37b8..636e84fe796 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -39,14 +39,14 @@ def test_anthropic_experimental_pass_through_messages_handler(): def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): """ - Test that api key, api base, and extra kwargs are forwarded to litellm.responses for Azure models. - Azure models are routed directly to the Responses API. + Test that api key, api base, and extra kwargs are forwarded to litellm.completion for Azure models. + Azure models are routed through chat/completions (not the Responses API). """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.responses", return_value="test-response") as mock_responses: + with patch("litellm.completion", return_value=MagicMock()) as mock_completion: try: anthropic_messages_handler( max_tokens=100, @@ -58,10 +58,10 @@ def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_an ) except Exception as e: print(f"Error: {e}") - mock_responses.assert_called_once() - assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" - assert mock_responses.call_args.kwargs["api_base"] == "test-api-base" - assert mock_responses.call_args.kwargs["custom_key"] == "custom_value" + mock_completion.assert_called_once() + assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" + assert mock_completion.call_args.kwargs["api_base"] == "test-api-base" + assert mock_completion.call_args.kwargs["custom_key"] == "custom_value" def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): From 15fcd90b9cf6f73f47c9abf6a73d1775461e987e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 27 Feb 2026 18:00:50 -0800 Subject: [PATCH 099/122] feat: add in_flight_requests metric to /health/backlog + prometheus (#22319) * feat: add in_flight_requests metric to /health/backlog + prometheus * refactor: clean class with static methods, add tests, fix sentinel pattern * docs: add in_flight_requests to prometheus metrics and latency troubleshooting --- docs/my-website/docs/proxy/prometheus.md | 25 +++++ .../docs/troubleshoot/latency_overhead.md | 34 ++++++- ...odel_prices_and_context_window_backup.json | 76 +++++++++++++- .../health_endpoints/_health_endpoints.py | 20 ++++ .../in_flight_requests_middleware.py | 81 +++++++++++++++ litellm/proxy/proxy_server.py | 4 + litellm/types/integrations/prometheus.py | 1 + .../test_in_flight_requests_middleware.py | 98 +++++++++++++++++++ 8 files changed, 334 insertions(+), 5 deletions(-) create mode 100644 litellm/proxy/middleware/in_flight_requests_middleware.py create mode 100644 tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 18a139d1d29..d8f0d83b59d 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -113,6 +113,31 @@ litellm_settings: ``` +## Pod Health Metrics + +Use these to measure per-pod queue depth and diagnose latency that occurs **before** LiteLLM starts processing a request. + +| Metric Name | Type | Description | +|---|---|---| +| `litellm_in_flight_requests` | Gauge | Number of HTTP requests currently in-flight on this uvicorn worker. Tracks the pod's queue depth in real time. With multiple workers, values are summed across all live workers (`livesum`). | + +### When to use this + +LiteLLM measures latency from when its handler starts. If a request waits in uvicorn's event loop before the handler runs, that wait is invisible to LiteLLM's own logs. `litellm_in_flight_requests` shows how loaded the pod was at any point in time. + +``` +high in_flight_requests + high ALB TargetResponseTime → pod overloaded, scale out +low in_flight_requests + high ALB TargetResponseTime → delay is pre-ASGI (event loop blocking) +``` + +You can also check the current value directly without Prometheus: + +```bash +curl http://localhost:4000/health/backlog \ + -H "Authorization: Bearer sk-..." +# {"in_flight_requests": 47} +``` + ## Proxy Level Tracking Metrics Use this to track overall LiteLLM Proxy usage. diff --git a/docs/my-website/docs/troubleshoot/latency_overhead.md b/docs/my-website/docs/troubleshoot/latency_overhead.md index cfb2cb43a7e..dd7f012dcde 100644 --- a/docs/my-website/docs/troubleshoot/latency_overhead.md +++ b/docs/my-website/docs/troubleshoot/latency_overhead.md @@ -2,9 +2,41 @@ Use this guide when you see unexpected latency overhead between LiteLLM proxy and the LLM provider. +## The Invisible Latency Gap + +LiteLLM measures latency from when its handler starts. If a request waits in uvicorn's event loop **before** the handler runs, that wait is invisible to LiteLLM's own logs. + +``` +T=0 Request arrives at load balancer + [queue wait — LiteLLM never logs this] +T=10 LiteLLM handler starts → timer begins +T=20 Response sent + +LiteLLM logs: 10s User experiences: 20s +``` + +To measure the pre-handler wait, poll `/health/backlog` on each pod: + +```bash +curl http://localhost:4000/health/backlog \ + -H "Authorization: Bearer sk-..." +# {"in_flight_requests": 47} +``` + +Or scrape the `litellm_in_flight_requests` Prometheus gauge at `/metrics`. + +| `in_flight_requests` | ALB `TargetResponseTime` | Diagnosis | +|---|---|---| +| High | High | Pod overloaded → scale out | +| Low | High | Delay is pre-ASGI — check for sync blocking code or event loop saturation | +| High | Normal | Pod is busy but healthy, no queue buildup | + +If you're on **AWS ALB**, correlate `litellm_in_flight_requests` spikes with ALB's `TargetResponseTime` CloudWatch metric. The gap between what ALB reports and what LiteLLM logs is the invisible wait. + ## Quick Checklist -1. **Collect the `x-litellm-overhead-duration-ms` response header** — this tells you LiteLLM's total overhead on every request. Start here. +1. **Check `in_flight_requests` on each pod** via `/health/backlog` or the `litellm_in_flight_requests` Prometheus gauge — this tells you if requests are queuing before LiteLLM starts processing. Start here for unexplained latency. +2. **Collect the `x-litellm-overhead-duration-ms` response header** — this tells you LiteLLM's total overhead on every request. 2. **Is DEBUG logging enabled?** This is the #1 cause of latency with large payloads. 3. **Are you sending large base64 payloads?** (images, PDFs) — see [Large Payload Overhead](#large-payload-overhead). 4. **Enable detailed timing headers** to pinpoint where time is spent. diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b21f23ac022..1ac8a347775 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -19210,6 +19210,39 @@ "supports_tool_choice": true, "supports_vision": false }, + "gpt-audio-1.5": { + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "gpt-audio-2025-08-28": { "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, @@ -20927,6 +20960,38 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-realtime-1.5": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, @@ -26650,8 +26715,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/swiss-ai/apertus-70b-instruct": { "input_cost_per_token": 0.0, @@ -26662,8 +26727,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/aisingapore/Gemma-SEA-LION-v4-27B-IT": { "input_cost_per_token": 0.0, @@ -32991,6 +33056,7 @@ "supports_web_search": true }, "xai/grok-2-vision-1212": { + "deprecation_date": "2026-02-28", "input_cost_per_image": 2e-06, "input_cost_per_token": 2e-06, "litellm_provider": "xai", @@ -33095,6 +33161,7 @@ }, "xai/grok-3-mini": { "cache_read_input_token_cost": 7.5e-08, + "deprecation_date": "2026-02-28", "input_cost_per_token": 3e-07, "litellm_provider": "xai", "max_input_tokens": 131072, @@ -33111,6 +33178,7 @@ }, "xai/grok-3-mini-beta": { "cache_read_input_token_cost": 7.5e-08, + "deprecation_date": "2026-02-28", "input_cost_per_token": 3e-07, "litellm_provider": "xai", "max_input_tokens": 131072, diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 4496ad92631..95b1836d8a9 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -33,6 +33,9 @@ from litellm.proxy.health_check import ( perform_health_check, run_with_timeout, ) +from litellm.proxy.middleware.in_flight_requests_middleware import ( + get_in_flight_requests, +) from litellm.secret_managers.main import get_secret #### Health ENDPOINTS #### @@ -1297,6 +1300,23 @@ async def health_readiness(): raise HTTPException(status_code=503, detail=f"Service Unhealthy ({str(e)})") +@router.get( + "/health/backlog", + tags=["health"], + dependencies=[Depends(user_api_key_auth)], +) +async def health_backlog(): + """ + Returns the number of HTTP requests currently in-flight on this uvicorn worker. + + Use this to measure per-pod queue depth. A high value means the worker is + processing many concurrent requests — requests arriving now will have to wait + for the event loop to get to them, adding latency before LiteLLM even starts + its own timer. + """ + return {"in_flight_requests": get_in_flight_requests()} + + @router.get( "/health/liveliness", # Historical LiteLLM name; doesn't match k8s terminology but kept for backwards compatibility tags=["health"], diff --git a/litellm/proxy/middleware/in_flight_requests_middleware.py b/litellm/proxy/middleware/in_flight_requests_middleware.py new file mode 100644 index 00000000000..d615640d870 --- /dev/null +++ b/litellm/proxy/middleware/in_flight_requests_middleware.py @@ -0,0 +1,81 @@ +""" +Tracks the number of HTTP requests currently in-flight on this uvicorn worker. + +Used by /health/backlog to expose per-pod queue depth, and emitted as the +Prometheus gauge `litellm_in_flight_requests`. +""" + +import os +from typing import Optional + +from starlette.types import ASGIApp, Receive, Scope, Send + + +class InFlightRequestsMiddleware: + """ + ASGI middleware that increments a counter when a request arrives and + decrements it when the response is sent (or an error occurs). + + The counter is class-level and therefore scoped to a single uvicorn worker + process — exactly the per-pod granularity we want. + + Also updates the `litellm_in_flight_requests` Prometheus gauge if + prometheus_client is installed. The gauge is lazily initialised on the + first request so that PROMETHEUS_MULTIPROC_DIR is already set by the time + we register the metric. Initialisation is attempted only once — if + prometheus_client is absent the class remembers and never retries. + """ + + _in_flight: int = 0 + _gauge: Optional[object] = None + _gauge_init_attempted: bool = False + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + InFlightRequestsMiddleware._in_flight += 1 + gauge = InFlightRequestsMiddleware._get_gauge() + if gauge is not None: + gauge.inc() # type: ignore[union-attr] + try: + await self.app(scope, receive, send) + finally: + InFlightRequestsMiddleware._in_flight -= 1 + if gauge is not None: + gauge.dec() # type: ignore[union-attr] + + @staticmethod + def get_count() -> int: + """Return the number of HTTP requests currently in-flight.""" + return InFlightRequestsMiddleware._in_flight + + @staticmethod + def _get_gauge() -> Optional[object]: + if InFlightRequestsMiddleware._gauge_init_attempted: + return InFlightRequestsMiddleware._gauge + InFlightRequestsMiddleware._gauge_init_attempted = True + try: + from prometheus_client import Gauge + + kwargs = {} + if "PROMETHEUS_MULTIPROC_DIR" in os.environ: + # livesum aggregates across all worker processes in the scrape response + kwargs["multiprocess_mode"] = "livesum" + InFlightRequestsMiddleware._gauge = Gauge( + "litellm_in_flight_requests", + "Number of HTTP requests currently in-flight on this uvicorn worker", + **kwargs, + ) + except Exception: + InFlightRequestsMiddleware._gauge = None + return InFlightRequestsMiddleware._gauge + + +def get_in_flight_requests() -> int: + """Module-level convenience wrapper used by the /health/backlog endpoint.""" + return InFlightRequestsMiddleware.get_count() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index be76c2ac5fb..48025863641 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -424,6 +424,9 @@ from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import ( router as user_agent_analytics_router, ) from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update +from litellm.proxy.middleware.in_flight_requests_middleware import ( + InFlightRequestsMiddleware, +) from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_router @@ -1404,6 +1407,7 @@ app.add_middleware( ) app.add_middleware(PrometheusAuthMiddleware) +app.add_middleware(InFlightRequestsMiddleware) def mount_swagger_ui(): diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 2c75276d9ca..0856d8a6f9b 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -237,6 +237,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_remaining_api_key_tokens_for_model", "litellm_llm_api_failed_requests_metric", "litellm_callback_logging_failures_metric", + "litellm_in_flight_requests", ] diff --git a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py new file mode 100644 index 00000000000..830bca49936 --- /dev/null +++ b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py @@ -0,0 +1,98 @@ +""" +Tests for InFlightRequestsMiddleware. + +Verifies that in_flight_requests is incremented during a request and +decremented after it completes, including on errors. +""" +import asyncio + +import pytest +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import JSONResponse, Response +from starlette.routing import Route +from starlette.testclient import TestClient + +from litellm.proxy.middleware.in_flight_requests_middleware import ( + InFlightRequestsMiddleware, + get_in_flight_requests, +) + + +@pytest.fixture(autouse=True) +def reset_state(): + """Reset class-level state between tests.""" + InFlightRequestsMiddleware._in_flight = 0 + yield + InFlightRequestsMiddleware._in_flight = 0 + + +def _make_app(handler): + from starlette.applications import Starlette + + app = Starlette(routes=[Route("/", handler)]) + app.add_middleware(InFlightRequestsMiddleware) + return app + + +# ── Structure ───────────────────────────────────────────────────────────────── + + +def test_is_not_base_http_middleware(): + """Must be pure ASGI — BaseHTTPMiddleware causes streaming degradation.""" + assert not issubclass(InFlightRequestsMiddleware, BaseHTTPMiddleware) + + +def test_has_asgi_call_protocol(): + assert "__call__" in InFlightRequestsMiddleware.__dict__ + + +# ── Counter behaviour ───────────────────────────────────────────────────────── + + +def test_counter_zero_at_start(): + assert get_in_flight_requests() == 0 + + +def test_counter_increments_inside_handler(): + captured = [] + + async def handler(request: Request) -> Response: + captured.append(InFlightRequestsMiddleware.get_count()) + return JSONResponse({}) + + TestClient(_make_app(handler)).get("/") + assert captured == [1] + + +def test_counter_returns_to_zero_after_request(): + async def handler(request: Request) -> Response: + return JSONResponse({}) + + TestClient(_make_app(handler)).get("/") + assert get_in_flight_requests() == 0 + + +def test_counter_decrements_after_error(): + """Counter must reach 0 even when the handler raises.""" + + async def handler(request: Request) -> Response: + return Response("boom", status_code=500) + + TestClient(_make_app(handler)).get("/") + assert get_in_flight_requests() == 0 + + +def test_non_http_scopes_not_counted(): + """Lifespan / websocket scopes must not touch the counter.""" + + class _InnerApp: + async def __call__(self, scope, receive, send): + pass + + mw = InFlightRequestsMiddleware(_InnerApp()) + + asyncio.get_event_loop().run_until_complete( + mw({"type": "lifespan"}, None, None) # type: ignore[arg-type] + ) + assert get_in_flight_requests() == 0 From d292ed77029abd1b0c39257feb2d5871a0a70e3e Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 27 Feb 2026 23:02:17 -0300 Subject: [PATCH 100/122] fix(db): add missing migration for LiteLLM_ClaudeCodePluginTable PR #22271 added the LiteLLM_ClaudeCodePluginTable model to schema.prisma but did not include a corresponding migration file, causing test_aaaasschema_migration_check to fail. Co-Authored-By: Claude Opus 4.6 --- .../migration.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260228000000_add_claude_code_plugin_table/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260228000000_add_claude_code_plugin_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260228000000_add_claude_code_plugin_table/migration.sql new file mode 100644 index 00000000000..e2a3694e8ef --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260228000000_add_claude_code_plugin_table/migration.sql @@ -0,0 +1,18 @@ +-- CreateTable +CREATE TABLE "LiteLLM_ClaudeCodePluginTable" ( + "id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "version" TEXT, + "description" TEXT, + "manifest_json" TEXT, + "files_json" TEXT DEFAULT '{}', + "enabled" BOOLEAN NOT NULL DEFAULT true, + "created_at" TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT, + + CONSTRAINT "LiteLLM_ClaudeCodePluginTable_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_ClaudeCodePluginTable_name_key" ON "LiteLLM_ClaudeCodePluginTable"("name"); From 2ac5365e06f5416c411571efc287e2c35a24e355 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 27 Feb 2026 23:04:43 -0300 Subject: [PATCH 101/122] fix: update stale docstring to match guardrail voicing behavior Addresses Greptile review feedback. Co-Authored-By: Claude Opus 4.6 --- .../test_litellm/litellm_core_utils/test_realtime_streaming.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 56c1cfa8515..7d38a5cc80a 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -379,7 +379,8 @@ async def test_realtime_guardrail_blocks_prompt_injection(): """ Test that when a transcription event containing prompt injection arrives from the backend, a registered guardrail blocks it — sending a warning to the client - and NOT sending response.create to the backend. + and voicing the guardrail violation message via response.cancel + + conversation.item.create + response.create. """ import litellm from litellm.integrations.custom_guardrail import CustomGuardrail From fb72979432a14f1dc31b164fd7eab3cd23e4054b Mon Sep 17 00:00:00 2001 From: Shivaang Date: Wed, 25 Feb 2026 23:14:43 -0500 Subject: [PATCH 102/122] fix(caching): store background task references in LLMClientCache._remove_key to prevent unawaited coroutine warnings Fixes #22128 --- litellm/caching/llm_caching_handler.py | 26 +++ .../caching/test_llm_caching_handler.py | 158 ++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 tests/test_litellm/caching/test_llm_caching_handler.py diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py index 16eb824f4c9..331aa8f51cd 100644 --- a/litellm/caching/llm_caching_handler.py +++ b/litellm/caching/llm_caching_handler.py @@ -3,11 +3,37 @@ Add the event loop to the cache key, to prevent event loop closed errors. """ import asyncio +from typing import Set from .in_memory_cache import InMemoryCache class LLMClientCache(InMemoryCache): + # Background tasks must be stored to prevent garbage collection, which would + # trigger "coroutine was never awaited" warnings. See: + # https://docs.python.org/3/library/asyncio-task.html#creating-tasks + # Intentionally shared across all instances as a global task registry. + _background_tasks: Set[asyncio.Task] = set() + + def _remove_key(self, key: str) -> None: + """Close async clients before evicting them to prevent connection pool leaks.""" + value = self.cache_dict.get(key) + super()._remove_key(key) + if value is not None: + close_fn = getattr(value, "aclose", None) or getattr(value, "close", None) + if close_fn and asyncio.iscoroutinefunction(close_fn): + try: + task = asyncio.get_running_loop().create_task(close_fn()) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) + except RuntimeError: + pass + elif close_fn and callable(close_fn): + try: + close_fn() + except Exception: + pass + def update_cache_key_with_event_loop(self, key): """ Add the event loop to the cache key, to prevent event loop closed errors. diff --git a/tests/test_litellm/caching/test_llm_caching_handler.py b/tests/test_litellm/caching/test_llm_caching_handler.py new file mode 100644 index 00000000000..0ac4ac5de79 --- /dev/null +++ b/tests/test_litellm/caching/test_llm_caching_handler.py @@ -0,0 +1,158 @@ +import asyncio +import os +import sys +import warnings + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +from litellm.caching.llm_caching_handler import LLMClientCache + + +class MockAsyncClient: + """Mock async HTTP client with an async close method.""" + + def __init__(self): + self.closed = False + + async def close(self): + self.closed = True + + +class MockSyncClient: + """Mock sync HTTP client with a sync close method.""" + + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + +@pytest.mark.asyncio +async def test_remove_key_no_unawaited_coroutine_warning(): + """ + Test that evicting an async client from LLMClientCache does not produce + 'coroutine was never awaited' warnings. + + Regression test for https://github.com/BerriAI/litellm/issues/22128 + """ + cache = LLMClientCache(max_size_in_memory=2) + + mock_client = MockAsyncClient() + cache.cache_dict["test-key"] = mock_client + cache.ttl_dict["test-key"] = 0 # expired + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + cache._remove_key("test-key") + # Let the event loop process the close task + await asyncio.sleep(0.1) + + coroutine_warnings = [ + w for w in caught_warnings if "coroutine" in str(w.message).lower() + ] + assert ( + len(coroutine_warnings) == 0 + ), f"Got unawaited coroutine warnings: {coroutine_warnings}" + + +@pytest.mark.asyncio +async def test_remove_key_closes_async_client(): + """ + Test that evicting an async client from the cache properly closes it. + """ + cache = LLMClientCache(max_size_in_memory=2) + + mock_client = MockAsyncClient() + cache.cache_dict["test-key"] = mock_client + cache.ttl_dict["test-key"] = 0 + + cache._remove_key("test-key") + # Let the event loop process the close task + await asyncio.sleep(0.1) + + assert mock_client.closed is True + assert "test-key" not in cache.cache_dict + assert "test-key" not in cache.ttl_dict + + +def test_remove_key_closes_sync_client(): + """ + Test that evicting a sync client from the cache properly closes it. + """ + cache = LLMClientCache(max_size_in_memory=2) + + mock_client = MockSyncClient() + cache.cache_dict["test-key"] = mock_client + cache.ttl_dict["test-key"] = 0 + + cache._remove_key("test-key") + + assert mock_client.closed is True + assert "test-key" not in cache.cache_dict + + +@pytest.mark.asyncio +async def test_eviction_closes_async_clients(): + """ + Test that cache eviction (when cache is full) properly closes async clients + without producing warnings. + """ + cache = LLMClientCache(max_size_in_memory=2, default_ttl=1) + + clients = [] + for i in range(2): + client = MockAsyncClient() + clients.append(client) + cache.set_cache(f"key-{i}", client) + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + # This should trigger eviction of one of the existing entries + cache.set_cache("key-new", "new-value") + await asyncio.sleep(0.1) + + coroutine_warnings = [ + w for w in caught_warnings if "coroutine" in str(w.message).lower() + ] + assert ( + len(coroutine_warnings) == 0 + ), f"Got unawaited coroutine warnings: {coroutine_warnings}" + + +def test_remove_key_no_event_loop(): + """ + Test that _remove_key doesn't raise when there's no running event loop + (falls through to the RuntimeError except branch). + """ + cache = LLMClientCache(max_size_in_memory=2) + + mock_client = MockAsyncClient() + cache.cache_dict["test-key"] = mock_client + cache.ttl_dict["test-key"] = 0 + + # Should not raise even though there's no running event loop + cache._remove_key("test-key") + assert "test-key" not in cache.cache_dict + + +@pytest.mark.asyncio +async def test_background_tasks_cleaned_up_after_completion(): + """ + Test that completed close tasks are removed from the _background_tasks set. + """ + cache = LLMClientCache(max_size_in_memory=2) + + mock_client = MockAsyncClient() + cache.cache_dict["test-key"] = mock_client + cache.ttl_dict["test-key"] = 0 + + cache._remove_key("test-key") + # Let the task complete + await asyncio.sleep(0.1) + + assert len(cache._background_tasks) == 0 From 8ce358e30328431898ec022a099146cb21c77ce9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 27 Feb 2026 18:32:39 -0800 Subject: [PATCH 103/122] [Feat] Agent RBAC Permission Fix - Ensure Internal Users cannot create agents (#22329) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: enforce RBAC on agent endpoints — block non-admin create/update/delete - Add /v1/agents/{agent_id} to agent_routes so internal users can access GET-by-ID (previously returned 403 due to missing route pattern) - Add _check_agent_management_permission() guard to POST, PUT, PATCH, DELETE agent endpoints — only PROXY_ADMIN may mutate agents - Add user_api_key_dict param to delete_agent so the role check works - Add comprehensive unit tests for RBAC enforcement across all roles Co-authored-by: Ishaan Jaff * fix: mock prisma_client in internal user get-agent-by-id test Co-authored-by: Ishaan Jaff * feat(ui): hide agent create/delete controls for non-admin users Match MCP servers pattern: wrap '+ Add New Agent' button in isAdmin conditional so internal users see a read-only agents view. Delete buttons in card and table were already gated. Update empty-state copy for non-admin users. Add 7 Vitest tests covering role-based visibility. Co-authored-by: Ishaan Jaff --------- Co-authored-by: Cursor Agent Co-authored-by: Ishaan Jaff --- litellm/proxy/_types.py | 1 + litellm/proxy/agent_endpoints/endpoints.py | 30 ++- .../proxy/agent_endpoints/test_endpoints.py | 181 ++++++++++++++++++ ui/litellm-dashboard/package-lock.json | 82 +------- .../src/components/agents.test.tsx | 71 +++++++ .../src/components/agents.tsx | 12 +- .../src/components/agents/agent_card_grid.tsx | 6 +- 7 files changed, 299 insertions(+), 84 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/agents.test.tsx diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index dfc2ba59d96..3af878f49d3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -431,6 +431,7 @@ class LiteLLMRoutes(enum.Enum): agent_routes = [ "/v1/agents", + "/v1/agents/{agent_id}", "/agents", "/a2a/{agent_id}", "/a2a/{agent_id}/message/send", diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index b411b81b434..65674d01be7 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -31,6 +31,23 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( router = APIRouter() +def _check_agent_management_permission(user_api_key_dict: UserAPIKeyAuth) -> None: + """ + Raises HTTP 403 if the caller does not have permission to create, update, + or delete agents. Only PROXY_ADMIN users are allowed to perform these + write operations. + """ + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail={ + "error": "Only proxy admins can create, update, or delete agents. Your role={}".format( + user_api_key_dict.user_role + ) + }, + ) + + @router.get( "/v1/agents", tags=["[beta] A2A Agents"], @@ -164,6 +181,8 @@ async def create_agent( """ from litellm.proxy.proxy_server import prisma_client + _check_agent_management_permission(user_api_key_dict) + if prisma_client is None: raise HTTPException(status_code=500, detail="Prisma client not initialized") @@ -302,6 +321,8 @@ async def update_agent( """ from litellm.proxy.proxy_server import prisma_client + _check_agent_management_permission(user_api_key_dict) + if prisma_client is None: raise HTTPException( status_code=500, detail=CommonProxyErrors.db_not_connected_error.value @@ -391,6 +412,8 @@ async def patch_agent( """ from litellm.proxy.proxy_server import prisma_client + _check_agent_management_permission(user_api_key_dict) + if prisma_client is None: raise HTTPException( status_code=500, detail=CommonProxyErrors.db_not_connected_error.value @@ -441,7 +464,10 @@ async def patch_agent( tags=["Agents"], dependencies=[Depends(user_api_key_auth)], ) -async def delete_agent(agent_id: str): +async def delete_agent( + agent_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ Delete an agent @@ -460,6 +486,8 @@ async def delete_agent(agent_id: str): """ from litellm.proxy.proxy_server import prisma_client + _check_agent_management_permission(user_api_key_dict) + if prisma_client is None: raise HTTPException(status_code=500, detail="Prisma client not initialized") diff --git a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py index 8cec3538077..fcf8f048190 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py @@ -7,6 +7,7 @@ from fastapi.testclient import TestClient from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.agent_endpoints import endpoints as agent_endpoints from litellm.proxy.agent_endpoints.endpoints import ( + _check_agent_management_permission, get_agent_daily_activity, router, user_api_key_auth, @@ -47,6 +48,16 @@ def _sample_agent_response( ) +def _make_app_with_role(role: LitellmUserRoles) -> TestClient: + """Create a TestClient where the auth dependency returns the given role.""" + test_app = FastAPI() + test_app.include_router(router) + test_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", user_role=role + ) + return TestClient(test_app) + + app = FastAPI() app.include_router(router) app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( @@ -258,3 +269,173 @@ async def test_get_agent_daily_activity_with_agent_names(monkeypatch): "agent-1": {"agent_name": "First Agent"}, "agent-2": {"agent_name": "Second Agent"}, } + + +# ---------- RBAC enforcement tests ---------- + + +class TestAgentRBACInternalUser: + """Internal users should be able to read agents but not create/update/delete.""" + + @pytest.fixture(autouse=True) + def _setup(self, monkeypatch): + self.internal_client = _make_app_with_role(LitellmUserRoles.INTERNAL_USER) + self.mock_registry = MagicMock() + monkeypatch.setattr(agent_endpoints, "AGENT_REGISTRY", self.mock_registry) + + def test_should_allow_internal_user_to_list_agents(self, monkeypatch): + self.mock_registry.get_agent_list = MagicMock(return_value=[]) + resp = self.internal_client.get( + "/v1/agents", headers={"Authorization": "Bearer k"} + ) + assert resp.status_code == 200 + + def test_should_allow_internal_user_to_get_agent_by_id(self, monkeypatch): + self.mock_registry.get_agent_by_id = MagicMock( + return_value=_sample_agent_response() + ) + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + resp = self.internal_client.get( + "/v1/agents/agent-123", headers={"Authorization": "Bearer k"} + ) + assert resp.status_code == 200 + + def test_should_block_internal_user_from_creating_agent(self): + resp = self.internal_client.post( + "/v1/agents", + json=_sample_agent_config(), + headers={"Authorization": "Bearer k"}, + ) + assert resp.status_code == 403 + assert "Only proxy admins" in resp.json()["detail"]["error"] + + def test_should_block_internal_user_from_updating_agent(self): + resp = self.internal_client.put( + "/v1/agents/agent-123", + json=_sample_agent_config(), + headers={"Authorization": "Bearer k"}, + ) + assert resp.status_code == 403 + + def test_should_block_internal_user_from_patching_agent(self): + resp = self.internal_client.patch( + "/v1/agents/agent-123", + json={"agent_name": "new-name"}, + headers={"Authorization": "Bearer k"}, + ) + assert resp.status_code == 403 + + def test_should_block_internal_user_from_deleting_agent(self): + resp = self.internal_client.delete( + "/v1/agents/agent-123", headers={"Authorization": "Bearer k"} + ) + assert resp.status_code == 403 + + +class TestAgentRBACInternalUserViewOnly: + """View-only internal users should only be able to read agents.""" + + @pytest.fixture(autouse=True) + def _setup(self, monkeypatch): + self.viewer_client = _make_app_with_role( + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + ) + self.mock_registry = MagicMock() + monkeypatch.setattr(agent_endpoints, "AGENT_REGISTRY", self.mock_registry) + + def test_should_allow_view_only_user_to_list_agents(self): + self.mock_registry.get_agent_list = MagicMock(return_value=[]) + resp = self.viewer_client.get( + "/v1/agents", headers={"Authorization": "Bearer k"} + ) + assert resp.status_code == 200 + + def test_should_block_view_only_user_from_creating_agent(self): + resp = self.viewer_client.post( + "/v1/agents", + json=_sample_agent_config(), + headers={"Authorization": "Bearer k"}, + ) + assert resp.status_code == 403 + + def test_should_block_view_only_user_from_deleting_agent(self): + resp = self.viewer_client.delete( + "/v1/agents/agent-123", headers={"Authorization": "Bearer k"} + ) + assert resp.status_code == 403 + + +class TestAgentRBACProxyAdmin: + """Proxy admins should have full CRUD access to agents.""" + + @pytest.fixture(autouse=True) + def _setup(self, monkeypatch): + self.admin_client = _make_app_with_role(LitellmUserRoles.PROXY_ADMIN) + self.mock_registry = MagicMock() + monkeypatch.setattr(agent_endpoints, "AGENT_REGISTRY", self.mock_registry) + + def test_should_allow_admin_to_create_agent(self, monkeypatch): + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + self.mock_registry.get_agent_by_name = MagicMock(return_value=None) + self.mock_registry.add_agent_to_db = AsyncMock( + return_value=_sample_agent_response() + ) + self.mock_registry.register_agent = MagicMock() + resp = self.admin_client.post( + "/v1/agents", + json=_sample_agent_config(), + headers={"Authorization": "Bearer k"}, + ) + assert resp.status_code == 200 + + def test_should_allow_admin_to_delete_agent(self): + existing = { + "agent_id": "agent-123", + "agent_name": "Existing Agent", + "agent_card_params": _sample_agent_card_params(), + } + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=existing + ) + self.mock_registry.delete_agent_from_db = AsyncMock() + self.mock_registry.deregister_agent = MagicMock() + resp = self.admin_client.delete( + "/v1/agents/agent-123", headers={"Authorization": "Bearer k"} + ) + assert resp.status_code == 200 + + +class TestCheckAgentManagementPermission: + """Unit tests for the _check_agent_management_permission helper.""" + + def test_should_allow_proxy_admin(self): + auth = UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + _check_agent_management_permission(auth) + + @pytest.mark.parametrize( + "role", + [ + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ], + ) + def test_should_block_non_admin_roles(self, role): + from fastapi import HTTPException + + auth = UserAPIKeyAuth(user_id="user", user_role=role) + with pytest.raises(HTTPException) as exc_info: + _check_agent_management_permission(auth) + assert exc_info.value.status_code == 403 + + +class TestAgentRoutesIncludesAgentIdPattern: + """Verify that agent_routes includes the {agent_id} pattern for route access.""" + + def test_should_include_agent_id_pattern(self): + from litellm.proxy._types import LiteLLMRoutes + + assert "/v1/agents/{agent_id}" in LiteLLMRoutes.agent_routes.value diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index fc2aa1599d3..503ed4a62a8 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -90,7 +90,6 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", "integrity": "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==", - "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -1772,7 +1771,6 @@ "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", @@ -1783,7 +1781,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.0.0" @@ -1793,14 +1790,12 @@ "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.31", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, "license": "MIT", "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", @@ -1978,7 +1973,6 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", @@ -1992,7 +1986,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -2002,7 +1995,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", @@ -2326,7 +2318,7 @@ "version": "1.58.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.1.tgz", "integrity": "sha512-6LdVIUERWxQMmUSSQi0I53GgCBYgM2RpGngCPY7hSeju+VrKjq3lvs7HpJoPbDiY5QM5EYRtRX5fvrinnMAz3w==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "playwright": "1.58.1" @@ -3431,14 +3423,12 @@ "version": "15.7.15", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", - "dev": true, "license": "MIT" }, "node_modules/@types/react": { "version": "18.2.48", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.48.tgz", "integrity": "sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w==", - "dev": true, "license": "MIT", "dependencies": { "@types/prop-types": "*", @@ -3480,7 +3470,6 @@ "version": "0.26.0", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.26.0.tgz", "integrity": "sha512-WFHp9YUJQ6CKshqoC37iOlHnQSmxNc795UhB26CyBBttrN9svdIrUjl/NjnNmfcwtncN0h/0PPAFWv9ovP8mLA==", - "dev": true, "license": "MIT" }, "node_modules/@types/unist": { @@ -4341,14 +4330,12 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", - "dev": true, "license": "MIT" }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, "license": "ISC", "dependencies": { "normalize-path": "^3.0.0", @@ -4362,7 +4349,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -4375,7 +4361,6 @@ "version": "5.0.2", "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", - "dev": true, "license": "MIT" }, "node_modules/argparse": { @@ -4747,7 +4732,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4773,7 +4757,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" @@ -4889,7 +4872,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -5013,7 +4995,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", - "dev": true, "license": "MIT", "dependencies": { "anymatch": "~3.1.2", @@ -5038,7 +5019,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -5114,7 +5094,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -5175,7 +5154,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", - "dev": true, "license": "MIT", "bin": { "cssesc": "bin/cssesc" @@ -5589,14 +5567,12 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", - "dev": true, "license": "Apache-2.0" }, "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", - "dev": true, "license": "MIT" }, "node_modules/doctrine": { @@ -6510,7 +6486,6 @@ "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", - "dev": true, "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -6543,7 +6518,6 @@ "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, "license": "MIT", "engines": { "node": ">=12.0.0" @@ -6581,7 +6555,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -6742,7 +6715,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -6893,7 +6865,6 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.3" @@ -7391,7 +7362,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, "license": "MIT", "dependencies": { "binary-extensions": "^2.0.0" @@ -7444,7 +7414,6 @@ "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", - "dev": true, "license": "MIT", "dependencies": { "hasown": "^2.0.2" @@ -7505,7 +7474,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -7551,7 +7519,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" @@ -7600,7 +7567,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" @@ -7877,7 +7843,6 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -8163,7 +8128,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", - "dev": true, "license": "MIT", "engines": { "node": ">=14" @@ -8176,7 +8140,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, "license": "MIT" }, "node_modules/locate-path": { @@ -8491,7 +8454,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -8943,7 +8905,6 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", @@ -8957,7 +8918,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -9072,7 +9032,6 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", - "dev": true, "license": "MIT", "dependencies": { "any-promise": "^1.0.0", @@ -9284,7 +9243,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -9303,7 +9261,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -9648,7 +9605,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, "license": "MIT" }, "node_modules/path-scurry": { @@ -9695,7 +9651,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -9708,7 +9663,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -9718,7 +9672,6 @@ "version": "4.0.7", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6" @@ -9728,7 +9681,7 @@ "version": "1.58.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.1.tgz", "integrity": "sha512-+2uTZHxSCcxjvGc5C891LrS1/NlxglGxzrC4seZiVjcYVQfUa87wBL6rTDqzGjuoWNjnBzRqKmF6zRYGMvQUaQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "playwright-core": "1.58.1" @@ -9747,7 +9700,7 @@ "version": "1.58.1", "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.1.tgz", "integrity": "sha512-bcWzOaTxcW+VOOGBCQgnaKToLJ65d6AqfLVKEWvexyS3AS6rbXl+xdpYRMGSRBClPvyj44njOWoxjNdL/H9UNg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" @@ -9770,7 +9723,6 @@ "version": "8.5.6", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9799,7 +9751,6 @@ "version": "15.1.0", "resolved": "https://registry.npmjs.org/postcss-import/-/postcss-import-15.1.0.tgz", "integrity": "sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==", - "dev": true, "license": "MIT", "dependencies": { "postcss-value-parser": "^4.0.0", @@ -9817,7 +9768,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.1.0.tgz", "integrity": "sha512-oIAOTqgIo7q2EOwbhb8UalYePMvYoIeRY2YKntdpFQXNosSu3vLrniGgmH9OKs/qAkfoj5oB3le/7mINW1LCfw==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9843,7 +9793,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9886,7 +9835,6 @@ "version": "6.2.0", "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.2.0.tgz", "integrity": "sha512-HQbt28KulC5AJzG+cZtj9kvKB93CFCdLvog1WFLf1D+xmMvPGlBstkpTEZfK5+AN9hfJocyBFCNiqyS48bpgzQ==", - "dev": true, "funding": [ { "type": "opencollective", @@ -9912,7 +9860,6 @@ "version": "6.1.2", "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", - "dev": true, "license": "MIT", "dependencies": { "cssesc": "^3.0.0", @@ -9926,7 +9873,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", - "dev": true, "license": "MIT" }, "node_modules/prelude-ls": { @@ -10040,7 +9986,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, "funding": [ { "type": "github", @@ -10829,7 +10774,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", - "dev": true, "license": "MIT", "dependencies": { "pify": "^2.3.0" @@ -10839,7 +10783,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, "license": "MIT", "dependencies": { "picomatch": "^2.2.1" @@ -10852,7 +10795,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -11117,7 +11059,6 @@ "version": "1.22.11", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", - "dev": true, "license": "MIT", "dependencies": { "is-core-module": "^2.16.1", @@ -11158,7 +11099,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", - "dev": true, "license": "MIT", "engines": { "iojs": ">=1.0.0", @@ -11214,7 +11154,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, "funding": [ { "type": "github", @@ -11855,7 +11794,6 @@ "version": "3.35.1", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", "integrity": "sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==", - "dev": true, "license": "MIT", "dependencies": { "@jridgewell/gen-mapping": "^0.3.2", @@ -11891,7 +11829,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -11927,7 +11864,6 @@ "version": "3.4.19", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.19.tgz", "integrity": "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==", - "dev": true, "license": "MIT", "dependencies": { "@alloc/quick-lru": "^5.2.0", @@ -11965,7 +11901,6 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -11982,7 +11917,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -12010,7 +11944,6 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", - "dev": true, "license": "MIT", "dependencies": { "any-promise": "^1.0.0" @@ -12020,7 +11953,6 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", - "dev": true, "license": "MIT", "dependencies": { "thenify": ">= 3.1.0 < 4" @@ -12062,7 +11994,6 @@ "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", - "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", @@ -12129,7 +12060,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" @@ -12217,7 +12147,6 @@ "version": "0.1.13", "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", - "dev": true, "license": "Apache-2.0" }, "node_modules/tsconfig-paths": { @@ -12334,7 +12263,7 @@ "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -12536,7 +12465,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/uuid": { @@ -12990,7 +12918,7 @@ "version": "8.19.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=10.0.0" diff --git a/ui/litellm-dashboard/src/components/agents.test.tsx b/ui/litellm-dashboard/src/components/agents.test.tsx new file mode 100644 index 00000000000..2d4c879dec0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/agents.test.tsx @@ -0,0 +1,71 @@ +import React from "react"; +import { render, screen, waitFor } from "@testing-library/react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import AgentsPanel from "./agents"; + +vi.mock("./networking", () => ({ + getAgentsList: vi.fn().mockResolvedValue({ agents: [] }), + deleteAgentCall: vi.fn(), + keyListCall: vi.fn().mockResolvedValue({ keys: [] }), +})); + +vi.mock("./agents/add_agent_form", () => ({ + default: () =>
, +})); + +vi.mock("./agents/agent_card_grid", () => ({ + default: ({ isAdmin }: { isAdmin: boolean }) => ( +
+ ), +})); + +vi.mock("./agents/agent_info", () => ({ + default: () =>
, +})); + +describe("AgentsPanel", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render the Agents panel title", async () => { + render(); + expect(screen.getByText("Agents")).toBeInTheDocument(); + }); + + it("should show Add New Agent button for admin users", async () => { + render(); + expect(screen.getByText("+ Add New Agent")).toBeInTheDocument(); + }); + + it("should show Add New Agent button for proxy_admin users", async () => { + render(); + expect(screen.getByText("+ Add New Agent")).toBeInTheDocument(); + }); + + it("should not show Add New Agent button for internal_user role", async () => { + render(); + expect(screen.queryByText("+ Add New Agent")).not.toBeInTheDocument(); + }); + + it("should not show Add New Agent button for internal_user_viewer role", async () => { + render(); + expect(screen.queryByText("+ Add New Agent")).not.toBeInTheDocument(); + }); + + it("should pass isAdmin=true to AgentCardGrid for admin role", async () => { + render(); + await waitFor(() => { + const grid = screen.getByTestId("agent-card-grid"); + expect(grid).toHaveAttribute("data-is-admin", "true"); + }); + }); + + it("should pass isAdmin=false to AgentCardGrid for internal user role", async () => { + render(); + await waitFor(() => { + const grid = screen.getByTestId("agent-card-grid"); + expect(grid).toHaveAttribute("data-is-admin", "false"); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/agents.tsx b/ui/litellm-dashboard/src/components/agents.tsx index 7ab6a5f0381..8dd9bc7d01c 100644 --- a/ui/litellm-dashboard/src/components/agents.tsx +++ b/ui/litellm-dashboard/src/components/agents.tsx @@ -141,11 +141,13 @@ const AgentsPanel: React.FC = ({ accessToken, userRole }) => { showIcon className="mb-3" /> -
- -
+ {isAdmin && ( +
+ +
+ )}
{selectedAgentId ? ( diff --git a/ui/litellm-dashboard/src/components/agents/agent_card_grid.tsx b/ui/litellm-dashboard/src/components/agents/agent_card_grid.tsx index 0ba7902f8b2..5e984cf220d 100644 --- a/ui/litellm-dashboard/src/components/agents/agent_card_grid.tsx +++ b/ui/litellm-dashboard/src/components/agents/agent_card_grid.tsx @@ -37,7 +37,11 @@ const AgentCardGrid: React.FC = ({ if (!agentsList || agentsList.length === 0) { return (
-

No agents found. Create one to get started.

+

+ {isAdmin + ? "No agents found. Create one to get started." + : "No agents found. Contact an admin to create agents."} +

); } From 594600dcb528c66fe918c85f3bc53c9f72fa5d31 Mon Sep 17 00:00:00 2001 From: milan-berri Date: Sat, 28 Feb 2026 05:11:29 +0200 Subject: [PATCH 104/122] fix: Add PROXY_ADMIN role to system user for key rotation (#21896) * fix: Add PROXY_ADMIN role to system user for key rotation The key rotation worker was failing with 'You are not authorized to regenerate this key' when rotating team keys. This was because the system user created by get_litellm_internal_jobs_user_api_key_auth() was missing the user_role field. Without user_role=PROXY_ADMIN, the system user couldn't bypass team permission checks in can_team_member_execute_key_management_endpoint(), causing authorization failures for team key rotation. This fix adds user_role=LitellmUserRoles.PROXY_ADMIN to the system user, allowing it to bypass team permission checks and successfully rotate keys for all teams. * test: Add unit test for system user PROXY_ADMIN role - Verify internal jobs system user has PROXY_ADMIN role - Critical for key rotation to bypass team permission checks - Regression test for PR #21896 --- litellm/proxy/_types.py | 1 + tests/test_litellm/proxy/test_proxy_types.py | 24 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3af878f49d3..d2b39c73be1 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2417,6 +2417,7 @@ class UserAPIKeyAuth( key_alias=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, team_alias="system", user_id="system", + user_role=LitellmUserRoles.PROXY_ADMIN, ) diff --git a/tests/test_litellm/proxy/test_proxy_types.py b/tests/test_litellm/proxy/test_proxy_types.py index 0e47134478b..ae2b7bbf24c 100644 --- a/tests/test_litellm/proxy/test_proxy_types.py +++ b/tests/test_litellm/proxy/test_proxy_types.py @@ -45,3 +45,27 @@ def test_audit_log_masking(): json_before_value = json.loads(audit_log.before_value) assert json_before_value["token"] == "1q2132r222" assert json_before_value["key"] == "sk-1*****7890" + + +def test_internal_jobs_user_has_proxy_admin_role(): + """ + Test that the internal jobs system user has PROXY_ADMIN role. + + This is critical for key rotation to work properly. The system user needs + PROXY_ADMIN role to bypass team permission checks in + TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint() + + Regression test for: https://github.com/BerriAI/litellm/pull/21896 + """ + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + + # Get the system user used for internal jobs like key rotation + system_user = UserAPIKeyAuth.get_litellm_internal_jobs_user_api_key_auth() + + # Verify the system user has PROXY_ADMIN role + assert system_user.user_role == LitellmUserRoles.PROXY_ADMIN + + # Verify other expected properties + assert system_user.user_id == "system" + assert system_user.team_id == "system" + assert system_user.team_alias == "system" From 3e60ca368253f388b496f3cddb611d9ab9cffbce Mon Sep 17 00:00:00 2001 From: milan-berri Date: Sat, 28 Feb 2026 05:12:16 +0200 Subject: [PATCH 105/122] fix: populate user_id and user_info for admin users in /user/info (#22239) * fix: populate user_id and user_info for admin users in /user/info endpoint Fixes #22179 When admin users call /user/info without a user_id parameter, the endpoint was returning null for both user_id and user_info fields. This broke budgeting tooling that relies on /user/info to look up current budget and spend. Changes: - Modified _get_user_info_for_proxy_admin() to accept user_api_key_dict parameter - Added logic to fetch admin's own user info from database - Updated function to return admin's user_id and user_info instead of null - Updated unit test to verify admin user_id is populated The fix ensures admin users get their own user information just like regular users. * test: make mock get_data signature match real method - Updated MockPrismaClientDB.get_data() to accept all parameters that the real method accepts - Makes mock more robust against future refactors - Added datetime and Union imports - Mock now returns None when user_id is not provided --- .../internal_user_endpoints.py | 22 +++++++-- tests/proxy_unit_tests/test_proxy_utils.py | 48 +++++++++++++++++-- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index e535ccaaa46..5a0a05114a3 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -614,7 +614,7 @@ async def user_info( user_id is None and user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN ): - return await _get_user_info_for_proxy_admin() + return await _get_user_info_for_proxy_admin(user_api_key_dict=user_api_key_dict) elif user_id is None: user_id = user_api_key_dict.user_id ## GET USER ROW ## @@ -714,7 +714,7 @@ async def user_info( raise handle_exception_on_proxy(e) -async def _get_user_info_for_proxy_admin(): +async def _get_user_info_for_proxy_admin(user_api_key_dict: UserAPIKeyAuth): """ Admin UI Endpoint - Returns All Teams and Keys when Proxy Admin is querying @@ -754,9 +754,23 @@ async def _get_user_info_for_proxy_admin(): _teams_in_db = [LiteLLM_TeamTable(**team) for team in _teams_in_db] _teams_in_db.sort(key=lambda x: (getattr(x, "team_alias", "") or "")) returned_keys = _process_keys_for_user_info(keys=keys_in_db, all_teams=_teams_in_db) + + # Get admin's own user_id and user_info + admin_user_id = user_api_key_dict.user_id + admin_user_info = None + + if admin_user_id is not None: + admin_user_info = await prisma_client.get_data(user_id=admin_user_id) + if admin_user_info is not None: + admin_user_info = ( + admin_user_info.model_dump() + if isinstance(admin_user_info, BaseModel) + else admin_user_info + ) + return UserInfoResponse( - user_id=None, - user_info=None, + user_id=admin_user_id, + user_info=admin_user_info, keys=returned_keys, teams=_teams_in_db, ) diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index cc1cc278392..fd97a38b41e 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -2,7 +2,8 @@ import asyncio import json import os import sys -from typing import Any, Dict, List, Optional +from datetime import datetime +from typing import Any, Dict, List, Optional, Union from unittest.mock import Mock import pytest @@ -1486,12 +1487,46 @@ class MockPrismaClientDB: mock_key_data, ): self.db = MockDb(mock_team_data, mock_key_data) + + async def get_data( + self, + token: Optional[Union[str, list]] = None, + user_id: Optional[str] = None, + user_id_list: Optional[list] = None, + team_id: Optional[str] = None, + team_id_list: Optional[list] = None, + key_val: Optional[dict] = None, + table_name: Optional[str] = None, + query_type: str = "find_unique", + expires: Optional[datetime] = None, + reset_at: Optional[datetime] = None, + offset: Optional[int] = None, + limit: Optional[int] = None, + ): + """Mock get_data method to return user info for admin""" + from litellm.proxy._types import LiteLLM_UserTable + + # Return a proper LiteLLM_UserTable object when querying by user_id + if user_id: + return LiteLLM_UserTable( + user_id=user_id, + user_role="proxy_admin", + spend=0.0, + max_budget=None, + ) + return None @pytest.mark.asyncio async def test_get_user_info_for_proxy_admin(mock_team_data, mock_key_data): # Patch the prisma_client import - from litellm.proxy._types import UserInfoResponse + from litellm.proxy._types import UserAPIKeyAuth, UserInfoResponse + + # Create a mock user_api_key_dict for admin user + mock_user_api_key_dict = UserAPIKeyAuth( + user_id="admin_user_123", + user_role="proxy_admin", + ) with patch( "litellm.proxy.proxy_server.prisma_client", @@ -1502,11 +1537,18 @@ async def test_get_user_info_for_proxy_admin(mock_team_data, mock_key_data): ) # Execute the function - result = await _get_user_info_for_proxy_admin() + result = await _get_user_info_for_proxy_admin( + user_api_key_dict=mock_user_api_key_dict + ) # Verify the result structure assert isinstance(result, UserInfoResponse) assert len(result.keys) == 2 + # Verify admin's user_id is populated + assert result.user_id == "admin_user_123" + # Verify admin's user_info is populated + assert result.user_info is not None + assert result.user_info["user_id"] == "admin_user_123" def test_custom_openid_response(): From d49abf857715c466713ec0a7484d94a60041a1c8 Mon Sep 17 00:00:00 2001 From: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Date: Fri, 27 Feb 2026 19:15:51 -0800 Subject: [PATCH 106/122] [Fix] Pass MCP auth headers from request into tool fetch for /v1/responses and chat completions (#22291) * fixed dynamic auth for /responses with mcp * fixed greptile concern --- litellm/responses/main.py | 17 +++++ .../responses/mcp/chat_completions_handler.py | 26 +++---- .../mcp/litellm_proxy_mcp_handler.py | 15 +++- .../mcp_tests/test_aresponses_api_with_mcp.py | 71 +++++++++++++++++++ .../mcp/test_chat_completions_handler.py | 66 +++++++++++++++++ 5 files changed, 181 insertions(+), 14 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 276e0cb9a04..a627531e994 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -177,6 +177,19 @@ async def aresponses_api_with_mcp( "litellm_metadata", {} ).get("user_api_key_auth") + # Extract MCP auth headers from request (for dynamic auth when fetching tools) + mcp_auth_header: Optional[str] = None + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None + secret_fields = kwargs.get("secret_fields") + if secret_fields and isinstance(secret_fields, dict): + from litellm.responses.utils import ResponsesAPIRequestUtils + + mcp_auth_header, mcp_server_auth_headers, _, _ = ( + ResponsesAPIRequestUtils.extract_mcp_headers_from_request( + secret_fields=secret_fields, tools=tools + ) + ) + # Get original MCP tools (for events) and OpenAI tools (for LLM) by reusing existing methods ( original_mcp_tools, @@ -185,6 +198,8 @@ async def aresponses_api_with_mcp( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, litellm_trace_id=kwargs.get("litellm_trace_id"), + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( original_mcp_tools @@ -370,6 +385,8 @@ async def aresponses_api_with_mcp( ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, ) final_response = ( LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response( diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index 377ce396457..bacc627cc84 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -120,7 +120,18 @@ async def acompletion_with_mcp( # noqa: PLR0915 (kwargs.get("metadata", {}) or {}).get("user_api_key_auth") ) - # Process MCP tools + # Extract MCP auth headers before fetching tools (needed for dynamic auth) + ( + mcp_auth_header, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + ) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request( + secret_fields=kwargs.get("secret_fields"), + tools=tools, + ) + + # Process MCP tools (pass auth headers for dynamic auth) ( deduplicated_mcp_tools, tool_server_map, @@ -128,6 +139,8 @@ async def acompletion_with_mcp( # noqa: PLR0915 user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, litellm_trace_id=kwargs.get("litellm_trace_id"), + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( @@ -143,17 +156,6 @@ async def acompletion_with_mcp( # noqa: PLR0915 mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy ) - # Extract MCP auth headers - ( - mcp_auth_header, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - ) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request( - secret_fields=kwargs.get("secret_fields"), - tools=tools, - ) - # Prepare call parameters # Remove keys that shouldn't be passed to acompletion clean_kwargs = {k: v for k, v in kwargs.items() if k not in ["acompletion"]} diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 805a1958552..5776ef95acb 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -99,6 +99,8 @@ class LiteLLM_Proxy_MCP_Handler: user_api_key_auth: Any, mcp_tools_with_litellm_proxy: Optional[Iterable[ToolParam]], litellm_trace_id: Optional[str] = None, + mcp_auth_header: Optional[str] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, ) -> tuple[List[MCPTool], List[str]]: """ Get available tools from the MCP server manager. @@ -106,6 +108,8 @@ class LiteLLM_Proxy_MCP_Handler: Args: user_api_key_auth: User authentication info for access control mcp_tools_with_litellm_proxy: ToolParam objects with server_url starting with "litellm_proxy" + mcp_auth_header: Optional deprecated auth header for MCP servers + mcp_server_auth_headers: Optional server-specific auth headers (e.g. from x-mcp-{alias}-*) Returns: List of MCP tools @@ -133,13 +137,14 @@ class LiteLLM_Proxy_MCP_Handler: tools = await _get_tools_from_mcp_servers( user_api_key_auth=user_api_key_auth, - mcp_auth_header=None, + mcp_auth_header=mcp_auth_header, mcp_servers=mcp_servers, - mcp_server_auth_headers=None, + mcp_server_auth_headers=mcp_server_auth_headers, log_list_tools_to_spendlogs=True, list_tools_log_source="responses", litellm_trace_id=litellm_trace_id, ) + allowed_mcp_server_ids = ( await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) ) @@ -278,6 +283,8 @@ class LiteLLM_Proxy_MCP_Handler: user_api_key_auth: Any, mcp_tools_with_litellm_proxy: List[ToolParam], litellm_trace_id: Optional[str] = None, + mcp_auth_header: Optional[str] = None, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None, ) -> tuple[List[Any], dict[str, str]]: """ Process MCP tools through filtering and deduplication pipeline without OpenAI transformation. @@ -286,6 +293,8 @@ class LiteLLM_Proxy_MCP_Handler: Args: user_api_key_auth: User authentication info for access control mcp_tools_with_litellm_proxy: ToolParam objects with server_url starting with "litellm_proxy" + mcp_auth_header: Optional deprecated auth header for MCP servers + mcp_server_auth_headers: Optional server-specific auth headers (e.g. from x-mcp-{alias}-*) Returns: List of filtered and deduplicated MCP tools in their original format @@ -301,6 +310,8 @@ class LiteLLM_Proxy_MCP_Handler: user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, litellm_trace_id=litellm_trace_id, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, ) # Step 2: Filter tools based on allowed_tools parameter diff --git a/tests/mcp_tests/test_aresponses_api_with_mcp.py b/tests/mcp_tests/test_aresponses_api_with_mcp.py index bae0b15dfec..c22c3537af8 100644 --- a/tests/mcp_tests/test_aresponses_api_with_mcp.py +++ b/tests/mcp_tests/test_aresponses_api_with_mcp.py @@ -3,6 +3,7 @@ import os import sys import pytest from typing import List, Any, cast +from unittest.mock import AsyncMock, patch sys.path.insert(0, os.path.abspath("../../..")) @@ -254,6 +255,76 @@ async def test_aresponses_api_with_mcp_mock_integration(): print(f"Other tools parsed: {len(other_parsed)}") +@pytest.mark.asyncio +async def test_aresponses_api_with_mcp_passes_mcp_server_auth_headers_to_process_tools(): + """ + Test that MCP auth headers from secret_fields (e.g. x-mcp-linear_config-authorization) + are passed to _process_mcp_tools_without_openai_transform when using the responses API. + """ + from litellm.responses.main import aresponses_api_with_mcp + + captured_process_kwargs = {} + + async def mock_process(**kwargs): + captured_process_kwargs.update(kwargs) + return ([], {}) + + mock_response = ResponsesAPIResponse( + **{ + "id": "resp_test", + "object": "response", + "created_at": 1234567890, + "status": "completed", + "error": None, + "incomplete_details": None, + "instructions": None, + "max_output_tokens": None, + "model": "gpt-4o", + "output": [{"type": "message", "id": "msg_1", "status": "completed", "role": "assistant", "content": []}], + "parallel_tool_calls": True, + "previous_response_id": None, + "reasoning": {"effort": None, "summary": None}, + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}}, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "truncation": "disabled", + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + "user": None, + "metadata": {}, + } + ) + + mcp_tools = [{"type": "mcp", "server_url": "litellm_proxy"}] + secret_fields = { + "raw_headers": {"x-mcp-linear_config-authorization": "Bearer linear-token"}, + } + + with patch.object( + LiteLLM_Proxy_MCP_Handler, + "_process_mcp_tools_without_openai_transform", + mock_process, + ), patch( + "litellm.responses.main.aresponses", + new_callable=AsyncMock, + return_value=mock_response, + ): + await aresponses_api_with_mcp( + input=[{"role": "user", "type": "message", "content": "hi"}], + model="gpt-4o", + tools=mcp_tools, + secret_fields=secret_fields, + ) + + assert "mcp_server_auth_headers" in captured_process_kwargs + mcp_server_auth_headers = captured_process_kwargs["mcp_server_auth_headers"] + assert mcp_server_auth_headers is not None + assert "linear_config" in mcp_server_auth_headers + assert mcp_server_auth_headers["linear_config"]["Authorization"] == "Bearer linear-token" + + @pytest.mark.asyncio async def test_mcp_allowed_tools_filtering(): """ diff --git a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py index a238531d2e0..3ba41705733 100644 --- a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py +++ b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py @@ -90,6 +90,72 @@ async def test_acompletion_with_mcp_without_auto_execution_calls_model(monkeypat assert captured_secret_fields["value"] == {"api_key": "value"} +@pytest.mark.asyncio +async def test_acompletion_with_mcp_passes_mcp_server_auth_headers_to_process_tools( + monkeypatch, +): + """ + Test that MCP auth headers extracted from secret_fields (e.g. x-mcp-linear_config-authorization) + are passed to _process_mcp_tools_without_openai_transform for dynamic auth when fetching tools. + """ + tools = [{"type": "mcp", "server_url": "litellm_proxy"}] + mock_acompletion = AsyncMock(return_value="ok") + + captured_process_kwargs = {} + + async def mock_process(**kwargs): + captured_process_kwargs.update(kwargs) + return ([], {}) + + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_should_use_litellm_mcp_gateway", + staticmethod(lambda t: True), + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_parse_mcp_tools", + staticmethod(lambda t: (t, [])), + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_process_mcp_tools_without_openai_transform", + mock_process, + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_transform_mcp_tools_to_openai", + staticmethod(lambda *_, **__: ["openai-tool"]), + ) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, + "_should_auto_execute_tools", + staticmethod(lambda **_: False), + ) + + # secret_fields with raw_headers containing MCP auth - extract_mcp_headers_from_request + # will parse these and pass to _process_mcp_tools_without_openai_transform + secret_fields = { + "raw_headers": { + "x-mcp-linear_config-authorization": "Bearer linear-token", + }, + } + + with patch("litellm.acompletion", mock_acompletion): + await acompletion_with_mcp( + model="test-model", + messages=[], + tools=tools, + secret_fields=secret_fields, + ) + + assert "mcp_server_auth_headers" in captured_process_kwargs + mcp_server_auth_headers = captured_process_kwargs["mcp_server_auth_headers"] + assert mcp_server_auth_headers is not None + assert "linear_config" in mcp_server_auth_headers + assert mcp_server_auth_headers["linear_config"]["Authorization"] == "Bearer linear-token" + + @pytest.mark.asyncio async def test_acompletion_with_mcp_auto_exec_performs_follow_up(monkeypatch): from litellm.utils import CustomStreamWrapper From fcabf9b602ade7a6986c46b360c2aa67b30fe27e Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 13 Feb 2026 06:24:41 -0300 Subject: [PATCH 107/122] fix(bedrock): filter internal json_tool_call when mixed with real tools Fixes #18381: When using both tools and response_format with Bedrock Converse API, LiteLLM internally adds json_tool_call to handle structured output. Bedrock may return both this internal tool AND real user-defined tools, breaking consumers like OpenAI Agents SDK. Changes: - Non-streaming: Added _filter_json_mode_tools() to handle 3 scenarios: only json_tool_call (convert to content), mixed (filter it out), or no json_tool_call (pass through) - Streaming: Added json_mode tracking to AWSEventStreamDecoder to suppress json_tool_call chunks and convert to text content - Fixed optional_params.pop() mutation issue Co-Authored-By: Claude Sonnet 4.5 --- litellm/llms/bedrock/chat/converse_handler.py | 2 +- .../bedrock/chat/converse_transformation.py | 115 +++++--- litellm/llms/bedrock/chat/invoke_handler.py | 64 +++-- .../chat/test_converse_transformation.py | 259 ++++++++++++++++++ 4 files changed, 391 insertions(+), 49 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 60a93b169c8..ec5b942ec1b 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -68,7 +68,7 @@ def make_sync_call( model_response=model_response, json_mode=json_mode ) else: - decoder = AWSEventStreamDecoder(model=model) + decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) # LOGGING diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index a0f2f65fb7f..1f48c290043 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1779,6 +1779,81 @@ class AmazonConverseConfig(BaseConfig): return content_str, tools, reasoningContentBlocks, citationsContentBlocks + @staticmethod + def _filter_json_mode_tools( + json_mode: Optional[bool], + tools: List[ChatCompletionToolCallChunk], + chat_completion_message: ChatCompletionResponseMessage, + ) -> Optional[List[ChatCompletionToolCallChunk]]: + """ + When json_mode is True, Bedrock may return the internal `json_tool_call` + tool alongside real user-defined tools. This method handles 3 scenarios: + + 1. Only json_tool_call present -> convert to text content, return None + 2. Mixed json_tool_call + real -> filter out json_tool_call, return real tools + 3. No json_tool_call / no json_mode -> return tools as-is + """ + if not json_mode or not tools: + return tools if tools else None + + json_tool_indices = [ + i + for i, t in enumerate(tools) + if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME + ] + + if not json_tool_indices: + # No json_tool_call found, return tools unchanged + return tools + + if len(json_tool_indices) == len(tools): + # All tools are json_tool_call — convert first one to content + verbose_logger.debug( + "Processing JSON tool call response for response_format" + ) + json_mode_content_str: Optional[str] = tools[0]["function"].get( + "arguments" + ) + if json_mode_content_str is not None: + try: + response_data = json.loads(json_mode_content_str) + if ( + isinstance(response_data, dict) + and "properties" in response_data + and len(response_data) == 1 + ): + response_data = response_data["properties"] + json_mode_content_str = json.dumps(response_data) + except json.JSONDecodeError: + pass + chat_completion_message["content"] = json_mode_content_str + return None + + # Mixed: filter out json_tool_call, keep real tools. + # Preserve the json_tool_call content as message text so the structured + # output from response_format is not silently lost. + first_idx = json_tool_indices[0] + json_mode_args = tools[first_idx]["function"].get("arguments") + if json_mode_args is not None: + try: + response_data = json.loads(json_mode_args) + if ( + isinstance(response_data, dict) + and "properties" in response_data + and len(response_data) == 1 + ): + response_data = response_data["properties"] + json_mode_args = json.dumps(response_data) + except json.JSONDecodeError: + pass + existing = chat_completion_message.get("content") or "" + chat_completion_message["content"] = ( + existing + json_mode_args if existing else json_mode_args + ) + + real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices] + return real_tools if real_tools else None + def _transform_response( # noqa: PLR0915 self, model: str, @@ -1801,7 +1876,7 @@ class AmazonConverseConfig(BaseConfig): additional_args={"complete_input_dict": data}, ) - json_mode: Optional[bool] = optional_params.pop("json_mode", None) + json_mode: Optional[bool] = optional_params.get("json_mode", None) ## RESPONSE OBJECT try: completion_response = ConverseResponseBlock(**response.json()) # type: ignore @@ -1885,37 +1960,13 @@ class AmazonConverseConfig(BaseConfig): self._transform_thinking_blocks(reasoningContentBlocks) ) chat_completion_message["content"] = content_str - if ( - json_mode is True - and tools is not None - and len(tools) == 1 - and tools[0]["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME - ): - verbose_logger.debug( - "Processing JSON tool call response for response_format" - ) - json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments") - if json_mode_content_str is not None: - # Bedrock returns the response wrapped in a "properties" object - # We need to extract the actual content from this wrapper - try: - response_data = json.loads(json_mode_content_str) - - # If Bedrock wrapped the response in "properties", extract the content - if ( - isinstance(response_data, dict) - and "properties" in response_data - and len(response_data) == 1 - ): - response_data = response_data["properties"] - json_mode_content_str = json.dumps(response_data) - except json.JSONDecodeError: - # If parsing fails, use the original response - pass - - chat_completion_message["content"] = json_mode_content_str - elif tools: - chat_completion_message["tool_calls"] = tools + filtered_tools = self._filter_json_mode_tools( + json_mode=json_mode, + tools=tools, + chat_completion_message=chat_completion_message, + ) + if filtered_tools: + chat_completion_message["tool_calls"] = filtered_tools ## CALCULATING USAGE - bedrock returns usage in the headers usage = self._transform_usage(completion_response["usage"]) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 1c58a11eebe..88f7341ed08 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -22,6 +22,7 @@ import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.caching.caching import InMemoryCache +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.logging_utils import track_llm_api_timing @@ -252,7 +253,7 @@ async def make_call( response.aiter_bytes(chunk_size=stream_chunk_size) ) else: - decoder = AWSEventStreamDecoder(model=model) + decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) completion_stream = decoder.aiter_bytes( response.aiter_bytes(chunk_size=stream_chunk_size) ) @@ -346,7 +347,7 @@ def make_sync_call( response.iter_bytes(chunk_size=stream_chunk_size) ) else: - decoder = AWSEventStreamDecoder(model=model) + decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) completion_stream = decoder.iter_bytes( response.iter_bytes(chunk_size=stream_chunk_size) ) @@ -1282,7 +1283,7 @@ def get_response_stream_shape(): class AWSEventStreamDecoder: - def __init__(self, model: str) -> None: + def __init__(self, model: str, json_mode: Optional[bool] = False) -> None: from botocore.parsers import EventStreamJSONParser self.model = model @@ -1290,6 +1291,8 @@ class AWSEventStreamDecoder: self.content_blocks: List[ContentBlockDeltaEvent] = [] self.tool_calls_index: Optional[int] = None self.response_id: Optional[str] = None + self.json_mode = json_mode + self._current_tool_name: Optional[str] = None def check_empty_tool_call_args(self) -> bool: """ @@ -1391,6 +1394,16 @@ class AWSEventStreamDecoder: response_tool_name = get_bedrock_tool_name( response_tool_name=_response_tool_name ) + self._current_tool_name = response_tool_name + + # When json_mode is True, suppress the internal json_tool_call + # and convert its content to text in delta events instead + if ( + self.json_mode is True + and response_tool_name == RESPONSE_FORMAT_TOOL_NAME + ): + return tool_use, provider_specific_fields, thinking_blocks + self.tool_calls_index = ( 0 if self.tool_calls_index is None else self.tool_calls_index + 1 ) @@ -1445,19 +1458,27 @@ class AWSEventStreamDecoder: if "text" in delta_obj: text = delta_obj["text"] elif "toolUse" in delta_obj: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": delta_obj["toolUse"]["input"], - }, - "index": ( - self.tool_calls_index - if self.tool_calls_index is not None - else index - ), - } + # When json_mode is True and this is the internal json_tool_call, + # convert tool input to text content instead of tool call arguments + if ( + self.json_mode is True + and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME + ): + text = delta_obj["toolUse"]["input"] + else: + tool_use = { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": delta_obj["toolUse"]["input"], + }, + "index": ( + self.tool_calls_index + if self.tool_calls_index is not None + else index + ), + } elif "reasoningContent" in delta_obj: provider_specific_fields = { "reasoningContent": delta_obj["reasoningContent"], @@ -1494,6 +1515,17 @@ class AWSEventStreamDecoder: ) -> Optional[ChatCompletionToolCallChunk]: """Handle stop/contentBlockIndex event in converse chunk parsing.""" tool_use: Optional[ChatCompletionToolCallChunk] = None + + # If the ending block was the internal json_tool_call, skip emitting + # the empty-args tool chunk and reset tracking state + if ( + self.json_mode is True + and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME + ): + self._current_tool_name = None + return tool_use + + self._current_tool_name = None is_empty = self.check_empty_tool_call_args() if is_empty: tool_use = { diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index f6d3d3c12f7..345f3ae7c5d 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -3493,3 +3493,262 @@ class TestBedrockMinThinkingBudgetTokens: drop_params=False, ) assert "thinking" not in result or result.get("thinking") is None + +def test_transform_response_with_both_json_tool_call_and_real_tool(): + """ + When Bedrock returns BOTH json_tool_call AND a real tool (get_weather), + only the real tool should remain in tool_calls. The json_tool_call should be filtered out. + Fixes https://github.com/BerriAI/litellm/issues/18381 + """ + from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig + from litellm.types.utils import ModelResponse + + response_json = { + "metrics": {"latencyMs": 200}, + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": { + "Current_Temperature": 62, + "Weather_Explanation": "Mild and cool.", + }, + } + }, + { + "toolUse": { + "toolUseId": "tooluse_weather_001", + "name": "get_weather", + "input": { + "location": "San Francisco, CA", + "unit": "fahrenheit", + }, + } + }, + ], + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 100, + "outputTokens": 50, + "totalTokens": 150, + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + }, + } + + class MockResponse: + def json(self): + return response_json + + @property + def text(self): + return json.dumps(response_json) + + config = AmazonConverseConfig() + model_response = ModelResponse() + optional_params = {"json_mode": True} + + result = config._transform_response( + model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + response=MockResponse(), + model_response=model_response, + stream=False, + logging_obj=None, + optional_params=optional_params, + api_key=None, + data=None, + messages=[], + encoding=None, + ) + + # Only real tool should remain + assert result.choices[0].message.tool_calls is not None + assert len(result.choices[0].message.tool_calls) == 1 + assert result.choices[0].message.tool_calls[0].function.name == "get_weather" + assert ( + result.choices[0].message.tool_calls[0].function.arguments + == '{"location": "San Francisco, CA", "unit": "fahrenheit"}' + ) + + # json_tool_call content should be preserved as message text + content = result.choices[0].message.content + assert content is not None + parsed = json.loads(content) + assert parsed["Current_Temperature"] == 62 + assert parsed["Weather_Explanation"] == "Mild and cool." + + +def test_transform_response_does_not_mutate_optional_params(): + """ + Verify that optional_params still contains json_mode after _transform_response. + Previously, .pop() was used which mutated the caller's dict. + """ + from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig + from litellm.types.utils import ModelResponse + + response_json = { + "metrics": {"latencyMs": 50}, + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_001", + "name": "json_tool_call", + "input": {"result": "ok"}, + } + } + ], + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 10, + "outputTokens": 5, + "totalTokens": 15, + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + }, + } + + class MockResponse: + def json(self): + return response_json + + @property + def text(self): + return json.dumps(response_json) + + config = AmazonConverseConfig() + model_response = ModelResponse() + optional_params = {"json_mode": True, "other_key": "value"} + + config._transform_response( + model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + response=MockResponse(), + model_response=model_response, + stream=False, + logging_obj=None, + optional_params=optional_params, + api_key=None, + data=None, + messages=[], + encoding=None, + ) + + # json_mode should still be in optional_params (not popped) + assert "json_mode" in optional_params + assert optional_params["json_mode"] is True + assert optional_params["other_key"] == "value" + + +def test_streaming_filters_json_tool_call_with_real_tools(): + """ + Simulate streaming chunks where both json_tool_call and a real tool arrive. + Verify json_tool_call chunks are converted to text content while real tool + chunks pass through normally. + """ + from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder + from litellm.types.llms.bedrock import ( + ContentBlockDeltaEvent, + ContentBlockStartEvent, + ) + + decoder = AWSEventStreamDecoder(model="test-model", json_mode=True) + + # Chunk 1: json_tool_call start + json_start = ContentBlockStartEvent( + toolUse={ + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + } + ) + tool_use_1, _, _ = decoder._handle_converse_start_event(json_start) + # json_tool_call start should be suppressed (return None tool_use) + assert tool_use_1 is None + # tool_calls_index should NOT have been incremented + assert decoder.tool_calls_index is None + + # Chunk 2: json_tool_call delta — should become text, not tool_use + json_delta = ContentBlockDeltaEvent(toolUse={"input": '{"temp": 62}'}) + text_2, tool_use_2, _, _, _ = decoder._handle_converse_delta_event( + json_delta, index=0 + ) + assert text_2 == '{"temp": 62}' + assert tool_use_2 is None + + # Chunk 3: json_tool_call stop + stop_tool = decoder._handle_converse_stop_event(index=0) + assert stop_tool is None + # _current_tool_name should be reset + assert decoder._current_tool_name is None + + # Chunk 4: real tool start + real_start = ContentBlockStartEvent( + toolUse={ + "toolUseId": "tooluse_weather_001", + "name": "get_weather", + } + ) + tool_use_4, _, _ = decoder._handle_converse_start_event(real_start) + assert tool_use_4 is not None + assert tool_use_4["function"]["name"] == "get_weather" + assert decoder.tool_calls_index == 0 + + # Chunk 5: real tool delta + real_delta = ContentBlockDeltaEvent( + toolUse={"input": '{"location": "SF"}'} + ) + text_5, tool_use_5, _, _, _ = decoder._handle_converse_delta_event( + real_delta, index=1 + ) + assert text_5 == "" + assert tool_use_5 is not None + assert tool_use_5["function"]["arguments"] == '{"location": "SF"}' + + +def test_streaming_without_json_mode_passes_all_tools(): + """ + Verify backward compatibility: when json_mode=False, all tools + (including json_tool_call if present) pass through unchanged. + """ + from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder + from litellm.types.llms.bedrock import ( + ContentBlockDeltaEvent, + ContentBlockStartEvent, + ) + + decoder = AWSEventStreamDecoder(model="test-model", json_mode=False) + + # json_tool_call start — should pass through when json_mode=False + json_start = ContentBlockStartEvent( + toolUse={ + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + } + ) + tool_use, _, _ = decoder._handle_converse_start_event(json_start) + assert tool_use is not None + assert tool_use["function"]["name"] == "json_tool_call" + assert decoder.tool_calls_index == 0 + + # json_tool_call delta — should be a tool_use, not text + json_delta = ContentBlockDeltaEvent(toolUse={"input": '{"data": 1}'}) + text, tool_use_delta, _, _, _ = decoder._handle_converse_delta_event( + json_delta, index=0 + ) + assert text == "" + assert tool_use_delta is not None + assert tool_use_delta["function"]["arguments"] == '{"data": 1}' + From b983352d11bd9c88de6765c492abdb1a88a312ed Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Sun, 15 Feb 2026 11:34:38 -0300 Subject: [PATCH 108/122] refactor: extract duplicated JSON unwrapping into helper method Addresses review comment from greptile-apps: https://github.com/BerriAI/litellm/pull/21107#pullrequestreview-3796085353 Changes: - Added `_unwrap_bedrock_properties()` helper method to eliminate code duplication - Replaced two identical JSON unwrapping blocks (lines 1592-1601 and 1612-1620) with calls to the new helper method - Improves maintainability - single source of truth for Bedrock properties unwrapping logic The helper method: - Parses JSON string - Checks for single "properties" key structure - Unwraps and returns the properties value - Returns original string if unwrapping not needed or parsing fails No functional changes - pure refactoring. Co-Authored-By: Claude Sonnet 4.5 --- .../bedrock/chat/converse_transformation.py | 55 +++++++++++-------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 1f48c290043..905c8453ef2 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1779,6 +1779,33 @@ class AmazonConverseConfig(BaseConfig): return content_str, tools, reasoningContentBlocks, citationsContentBlocks + @staticmethod + def _unwrap_bedrock_properties(json_str: str) -> str: + """ + Unwrap Bedrock's response_format JSON structure. + + If the JSON has a single "properties" key, extract its value. + Otherwise, return the original string. + + Args: + json_str: JSON string to unwrap + + Returns: + Unwrapped JSON string or original if unwrapping not needed + """ + try: + response_data = json.loads(json_str) + if ( + isinstance(response_data, dict) + and "properties" in response_data + and len(response_data) == 1 + ): + response_data = response_data["properties"] + return json.dumps(response_data) + except json.JSONDecodeError: + pass + return json_str + @staticmethod def _filter_json_mode_tools( json_mode: Optional[bool], @@ -1815,17 +1842,9 @@ class AmazonConverseConfig(BaseConfig): "arguments" ) if json_mode_content_str is not None: - try: - response_data = json.loads(json_mode_content_str) - if ( - isinstance(response_data, dict) - and "properties" in response_data - and len(response_data) == 1 - ): - response_data = response_data["properties"] - json_mode_content_str = json.dumps(response_data) - except json.JSONDecodeError: - pass + json_mode_content_str = BedrockConverseConfig._unwrap_bedrock_properties( + json_mode_content_str + ) chat_completion_message["content"] = json_mode_content_str return None @@ -1835,17 +1854,9 @@ class AmazonConverseConfig(BaseConfig): first_idx = json_tool_indices[0] json_mode_args = tools[first_idx]["function"].get("arguments") if json_mode_args is not None: - try: - response_data = json.loads(json_mode_args) - if ( - isinstance(response_data, dict) - and "properties" in response_data - and len(response_data) == 1 - ): - response_data = response_data["properties"] - json_mode_args = json.dumps(response_data) - except json.JSONDecodeError: - pass + json_mode_args = BedrockConverseConfig._unwrap_bedrock_properties( + json_mode_args + ) existing = chat_completion_message.get("content") or "" chat_completion_message["content"] = ( existing + json_mode_args if existing else json_mode_args From fdf4c9dbdcb91cf7c6aae19f9eab82955c47ac85 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Sun, 15 Feb 2026 11:43:43 -0300 Subject: [PATCH 109/122] fix: use correct class name AmazonConverseConfig in helper method calls Fixed MyPy errors where BedrockConverseConfig was used instead of AmazonConverseConfig in the _unwrap_bedrock_properties() calls. Errors: - Line 1619: BedrockConverseConfig -> AmazonConverseConfig - Line 1631: BedrockConverseConfig -> AmazonConverseConfig Co-Authored-By: Claude Sonnet 4.5 --- litellm/llms/bedrock/chat/converse_transformation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 905c8453ef2..d210f294c64 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1842,7 +1842,7 @@ class AmazonConverseConfig(BaseConfig): "arguments" ) if json_mode_content_str is not None: - json_mode_content_str = BedrockConverseConfig._unwrap_bedrock_properties( + json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties( json_mode_content_str ) chat_completion_message["content"] = json_mode_content_str @@ -1854,7 +1854,7 @@ class AmazonConverseConfig(BaseConfig): first_idx = json_tool_indices[0] json_mode_args = tools[first_idx]["function"].get("arguments") if json_mode_args is not None: - json_mode_args = BedrockConverseConfig._unwrap_bedrock_properties( + json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties( json_mode_args ) existing = chat_completion_message.get("content") or "" From 37ec9f89956b8ea2651b92d102dcf3938357f9b0 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 24 Feb 2026 22:32:41 +0000 Subject: [PATCH 110/122] fix: shorten guardrail benchmark result filenames for Windows long path support Fixes #21941 The generated result filenames from _save_confusion_results contained parentheses, dots, and full yaml filenames, producing paths that exceed the Windows 260-char MAX_PATH limit. Rework the safe_label logic to produce short {topic}_{method_abbrev} filenames (e.g. insults_cf.json) while preserving the full label inside the JSON content. Rename existing tracked result files to match the new naming convention. --- ...ied_insults.yaml).json => insults_cf.json} | 0 ...l_advice.yaml).json => investment_cf.json} | 0 .../guardrail_benchmarks/test_eval.py | 21 ++++++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_insults_-_contentfilter_(denied_insults.yaml).json => insults_cf.json} (100%) rename litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/{block_investment_-_contentfilter_(denied_financial_advice.yaml).json => investment_cf.json} (100%) diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_insults_-_contentfilter_(denied_insults.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/insults_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_insults_-_contentfilter_(denied_insults.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/insults_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_investment_-_contentfilter_(denied_financial_advice.yaml).json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/investment_cf.json similarity index 100% rename from litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/block_investment_-_contentfilter_(denied_financial_advice.yaml).json rename to litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/results/investment_cf.json diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py index 01e820163fd..9cce9dc1686 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py @@ -18,6 +18,7 @@ Run a specific eval: import json import os +import re import time from datetime import datetime, timezone from typing import Any, Dict, List @@ -105,7 +106,25 @@ def _print_confusion_report(label: str, metrics: dict, wrong: list) -> None: def _save_confusion_results(label: str, metrics: dict, wrong: list, rows: list) -> dict: """Save confusion matrix results to a JSON file and return the result dict.""" os.makedirs(RESULTS_DIR, exist_ok=True) - safe_label = label.lower().replace(" ", "_").replace("—", "-") + # Build a short, filesystem-safe filename from the label. + # Full label is preserved inside the JSON; filename just needs to be + # unique and recognisable. Format: {topic}_{method_abbrev}.json + parts = label.split("\u2014") + topic = parts[0].strip().lower().replace("block ", "").replace(" ", "_") + method_full = parts[1].strip() if len(parts) > 1 else "" + method_name = re.sub(r"\s*\(.*?\)", "", method_full).strip().lower() + qualifier_match = re.search(r"\(([^)]+)\)", method_full) + qualifier = qualifier_match.group(1) if qualifier_match else "" + qualifier = re.sub(r"\.[a-z]+$", "", qualifier) # drop .yaml etc. + if method_name == "contentfilter": + safe_label = f"{topic}_cf" + elif qualifier: + safe_label = f"{topic}_{method_name}_{qualifier}" + else: + safe_label = f"{topic}_{method_name}" + safe_label = safe_label.replace(" ", "_") + safe_label = re.sub(r"[^a-z0-9_\-]", "", safe_label) + safe_label = re.sub(r"_+", "_", safe_label).strip("_") result = { "label": label, "timestamp": datetime.now(timezone.utc).isoformat(), From bcf9acf5ea8cc7b026f2d3d3b272051fc9bc4ffc Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 24 Feb 2026 18:45:05 -0500 Subject: [PATCH 111/122] Update litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../litellm_content_filter/guardrail_benchmarks/test_eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py index 9cce9dc1686..ca66b4da652 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks/test_eval.py @@ -123,7 +123,7 @@ def _save_confusion_results(label: str, metrics: dict, wrong: list, rows: list) else: safe_label = f"{topic}_{method_name}" safe_label = safe_label.replace(" ", "_") - safe_label = re.sub(r"[^a-z0-9_\-]", "", safe_label) + safe_label = re.sub(r"[^a-z0-9_.\-]", "", safe_label) safe_label = re.sub(r"_+", "_", safe_label).strip("_") result = { "label": label, From bffce842a14ed7bf9995e0410fc1cb66a7aca23a Mon Sep 17 00:00:00 2001 From: rasmi Date: Fri, 27 Feb 2026 22:33:55 -0500 Subject: [PATCH 112/122] Remove Apache 2 license from SKILL.md (#22322) --- .../test_skills_data/slack-gif-creator.zip | Bin 16748 -> 12585 bytes .../slack-gif-creator/LICENSE.txt | 202 ------------------ .../slack-gif-creator/SKILL.md | 1 - 3 files changed, 203 deletions(-) delete mode 100644 tests/llm_translation/test_skills_data/slack-gif-creator/LICENSE.txt diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator.zip b/tests/llm_translation/test_skills_data/slack-gif-creator.zip index 15c60e3667d6d62e9c66b17977f7d25889a91571..9827db9ac2d98abf9de63ae1f38a9c31d19cee70 100644 GIT binary patch delta 4105 zcma*qWmJ@F*9Y(!Qex$mpa*Z%h3i3C>KQyJ zzP4*cE**bd|3C@=kVqpA;g5P(0DnTXEYl^F`7c=i zLy(y0lA!oWZvMtJdXp@W{Dq7_xB36M5PKw{e;?w zdglW?bHTHgTT0$Ki5_w1x>gpAGX%0`-v$+dgxSKe{YmNy2EruOX%P=(e^jXr^zSf10adBTA-@|QYa%` zAz2>b@&#QHzlxk|!QPu1R@&Y5S8iK+^yBn7YiIm!`V5WwT~px=Uah{*GpyNEhpoMV zxw^gh(o<`x(rB4{L_n-fx@Be#-L_9=YTa6tkA7rg-PUP&yv+* z_|V+9xg1RjUqiUFfR#`pK{lJU;PN}6^bkh``Td;P41)zh%g}?9m6_zsiTfECYHtN` z<_3GRCUpQe9ipoQxlHl#qL^Cf;PAVdFUB+yVYpbEC`Bnr^wc%To3NjU4m`&E2};+k z#`5#^dd_4yhi%%fCI>oE)yyaJb|#&d=u+#BCzMBW#uvNnaO<*aC~8zyh_R1sLuM)7 zf6$%B7{*~fGW3guJdDIuY(yN+i6Rln+Bt&?HizDeg$BKiInC+8u>9lJY1HW~JX0!m zW;5O;WHGRT1y=pcC#U|xDlN_(AX?bS_p`HCU_M+CGdikloKERkjR}yuPaQP0q+)RD z>;RDeWyP%C^(89B>d+$!SC+xe8m6j3hOpHr&@CRE8ej{Hy*Tk@AhigBeMb-|%5$2) zRlL^YI)<7156nxxw#>m4;(J_q! z7k;)R^WbAuaszqJtyS=?c|_$=KlJ7CCa7oqlLL*!Jl|VG=;-I??$XxgD_ux3me|wX z+{d`$L{htpQip3P7jie!v?EjVS*Pgx(N0x@e!F*mY%iX|*bjnl_g|cP(re%CReLC4 zFHh?V<2lPBus;zjiE`5Y*{73edjP*aB%9{o?-nIg31XD|dhE(dRfHI)(m23MSwXt7 zZnu7lftdoz?AAX$Scmj_6u{n3YK(@$`jnT6O`2l}iHrzX=9ez=TkOqL4kWSh0_fs} zo}rf#CB;pZ7~v+X;)esntm`mqP3mls#kd6{xfyQnX}F$ihT5#$exCNCi;M_TEViJP z_>G#gLlh*ip#-$>!@IgBt3?H7;P6NCjFC4Ob|4LEj&0^T<&8NfzzMIF zo>6utUfN!e{)k69RK_1#SJ^B|b$DfHDA9DhGepgJ$563CW<~O$2!OcbS-imjVA-cW@RH$jXG ze!WGNj;#)JZ>^1{AB(5jHoB45x>kLbw|HGQni1$z{;OWk)>sCnqN(42zBtqMP2r$@ z&L#7nSdNb|Kt@J-^9l`dR+OjL$K|xFRp7Wrg^{7~7PUxS0c#SjZgxw?3_BTR{zWHJ!Mz!gL&3YCQXl zS+b)%y>f>p=vhzea_|z(JyI9@dewA#l9ev?kOuE(i8m4EVISCT@0GB=G?_%!aHQ>& zg@f)okCP+oi<--12`0qSY|WB`vWOn@C1Qjsocl0ORq)Tt1XFh4dsmvbb|n&Ch=}_G z{dGFozma6NQ^aBF8FC?>c4Py0^CU1)DUBSP+(DHsUJs_b$b>J{trElzI{`SQENUn@ z?jvRG@DySctnVge2-wLEPFrO-e&bFEW!mcGjb38LF#-cNaVS@xqHy-7-Aqg%xoo@Y z3!8$inKX7g72{yD1g;uCyUOiprVJmuuWbZ+)bt+S5lo=6cBf2|nB`GDE-Oj6QDb?3g3^$tF@z1HU7U3)hUv?BZ73 z5#n|ip(NdB=jGR$>$JGMkx{|zys=R_YDZ^pVdKLm8%rkMqm%chZ>}hf>!mc+2@K^- z+Z0ek*@6$wD4EX*$fPhGGM$K*PxAAKwB0wvimtR7iup5&*EIz9+@77fz!;r2JTi*N zMK^k>`lr`CUhG&V*&{WbstQoca%GVe)<8&0?%gw$!>x@{)`w?}2bJDAXGF9aV6Xv0 z{HgxI)&pC&oP8(BfHDbgnXx>Q75MJtRE9IM-7^b|0ZPh=8yl9RjuoEt2fbq-Dx$dW zR7CvZuiZO~5*scmqr1KVfucx&4nIA(x?}o}+n@R@Di2Q}OFXH>Th;NVxunGMi-;XV zT75*B90m2S<~f?HQrn+^&T4n@_VZhkfIe_RoT)o8vXzBhj6=eM0-xjUH})LLMcq`e zXFp=Us=6CSrMkyw8M(9}mr zLpWcN70Qaf(i-(C!=Z8IB+XsC4`G|WW^4MW=R5n`vxe!ka1jqp;7tw~iVIr4>rJst zt$x<|CXQJKO2z2KbknVJn59&fK$CPeIPz*l=gULV=xC?L{pOgwv9%0SKwdXk@CqJT zEOvUeWKKEK@BqrtqW{e9mh<EAa#u&J400GF;nc6&Wb^-R%NNFt6Lrc-q{1MUQlL zw3w2QPvAu-gyM@G)!`9lVE7lIeP!2yiwVxd7w?oL3ejr3^a+I7F~@wAf&06%?U2Ne zYH}unq<|;{9NpIAcFTlxTQFwY9Vg23W4}9CrHAaK&)M1+uPSA7wajLU%g?>{I!Y-|Ar&eNF$_Q0vFht)X*ZG%dM#!HbM z&LqxPn0VKvBC?X4wt!Lx=F5=O)lXK7ao^6|uHuiRIFav`MR)iPWjqbmAUUrHiF2?J z!wyzDXlt56vEJ?a)UnnvP}|F2UEtZtFob@f{(KEfUm|B;EFqT$-md`{W0%oZuxQ6~ zr)#k5RjnU|$Tc?4GV3U5c81A;b^UC;AWzmsgnjd;3h-B%WoBT(QPfWeh0+hpPhH|% z*&g9WHIG`rqf_#J)cGy6re|&pz7&ZAKiZhL19IWf8{b1Re$gPn(xoPKNQt| z@BX8&y^`j?awAGQ|H{cLtBL+U@BP0~6r(IH^pA@qCb^9IWPf}qgaAOa*yZ2<0Ey2j Aod5s; delta 8268 zcmai(RahL|mWCUL#@*dLxJ%>i?gV$&5E|DIr19Vu+}$BK37Uow9D+MRLqq1zITvSU z&VOdAp1P=~-u>i-v{jVZ*0NcML@0N|h6{<-Bp^6D`~|5M%zW*z!JH*Fx5_)ic#**he} zKgR({qZT6ia~}U*J!F$w4)YJA+cSYGfC&H;LtfD^fhqqM29z6{`m72Pg&s6WLr`?9 z8CF$z7w6wmQyl2bG9@i*d|y91HNrttu2hadlAb(n*u0?FFD5g!cOg z=W9SzGM#D7Xj^)Y6dFF}=r53e_%UT?ux#mI zH!&~EF{5|3t`_2@a@tXim!!LdRuk%x{?jslEuR_q_Txy4&W;D#(ox$hmn-a4^4AZA z2ulTw(KC8`MTuZ^-KFSjl|VaHW?x20GmGBmqy4kbn@_gDe8$;bIJrwYHVzSo^NUtu zfkb7+n`*k$b5Vg8`O)o>y9t+x+B!Buy)1^^EAk{&6oIbMz&y3l)dg5Z&Vo$d47hW= zL8)#^E#9wdO3~=2wg~f(ywUA-EaVLguZi7h8!JT&?+rM>4}pz&k(il|^RtC#x(JQ4ccqQX5wNMC1_TD;Z%=sYsG48CmjcGM^k%c*9yH5IfEU}+9_*Gyxd)lB07v!lQs zwQCS{O$YE$wu+pnvg(DaG~4>gn#U~7o}6pOhU?o-i{M-Mn?z2so4$xoU_MpO*m4r{2hi7QRu8_r zLO#4E%rMi#)6D0s(-t>rY6AxcnkuE3dxIs@{H9oOhl`?*A^dpJ`Jq+l^xM}UMXoM0 zdK9qMJ`UzUaZ!|1BfXZ)f~|8I@xUk;CKA_ID_#-|S4f%6`bg91BWJmv_$iP>Cve@B zu3!`}Ms-%Oy#YlH?vEVpi=K4QnNnqJdDu`52q#Z5`kN!M84u}cP; zk|Qt^-`TtHu`OdWppol8Zek*fdWv*?1%WIrh2URaj4&c z2!)t5)6R2Nb1Np~K@C=bvvxK_bCz#nwuw`LO=%hqd{^`(+Y|A1E8I$S3FLXX>$K6H z$5B?gUKXc&aOoMu(?yYwX=UF^LbSX_?7kWRfFT0-g{g4EZH_fFESDK>>R@wfc#V>; zi$;wi^vE+m%}!fgY-zp2D$WAcRuaEO1beGDhR>RUT*zI>*gB8WtAGEu^n z40>N;YKT!Rq=^H5p$Y(1J z+B1bAp?tPm0xNtLXi9C4}EU_!AQt3hGWQFers$ehRZ}Xi}n`Yv64?7RO}rsH8>ndrwIuZ3YjGz zKA}OsOg zt>&3clO}GS$c;#_|C~VTJK&zMGw8mD6;`uBRdU#MtwXHULGT&mp|#QzHbJI3wDOZE zGprX%`bqRqcr^_x4ZK{9?qJWES9E~rRIjYJ0ggHG#J{C zpsZnS`CX94wwOvy=kP6(1+M1-mSh(lGjJkY#MOGfXoz&_*KefiQ1nS$B8Ikr@`o!G z;>>P$HxMf^a&q83?%I>85%$o#5Bc3+!u7xTpVK0TaD`&2(}PjlMA!T@i-r^HA|7vY zIgQwmu+fRyB43Vh4SpqPBy#QG|4{5JC_?Jq^M53Iflybf^vBQe9oj<32op+V4P6jQ zF_g;`_h_pTT4QwR2@J=oSlC&9_kmbJ?Xj=WqJjc_6`Akw230aN3(XgME($PxZw+`w zLzeZ7Q9oDmjDly2>pm=yl$|VihRWpL6@iw@FzC(}gMmG!lwL0ifzk@EDe_;CkHq17 zAuDKheTyr~JH;5B@8wkraMs_jX&8V1*{uLW>7EQHx|0)G{KaveP|5F4#X?2x-2gAFP}jKMvX(lb!w&f;z*!#?*-FNFJ2 zHfLxL-LxM$FPkDi@41G)=+^!WMpIbkTu`~*3nn;3sTLYA#gy##Mt|a%%h6{64Nf@0 zH6xaexAOi|k|qHR?F?pYJb{5R&vj@rv#Pbpv{O@fpp62m)IxYtw^)BtG|iS*XH=Mb z6UBrH51zPNmyE&GJbNRs*Bzc+%BZ|aQ4o-HE>A&@B)0CVjr$&4A&QRu+-iYO!dJ5i zaFjQ_f4zFl3mAJ<6sPZombOYt!AZ&Ej((rxmRabp;)r#3?>m9 zhb$?W3H3(o`g7iQZxAOW?0(sB2*>pk;olpoZ-c*HCF?yIyfX7fSgp)_OICy06TO*k z+p=?+?Z!Q3l{#jp?>~4;F#u~`N(p0kSAr9ul!H*qd_40)-@aFX&2+^P3(H^;dJ+k^ zR(&Qd7lBuL^W#kfN%&C4^_%Eu<`(MCnEaXjbbUbnr&lbf_qnB+}ZSZToy#bV&ML03ovrFQ(&8@0ks0<|NRKAAE1QW~2) z+QoasSSyx9WfuJzRPalKQJ@-w#C`Xcx^4r^+&-`_ezv^Fl+ZHIj}vVTFrr-GqSR8 zgO9B!N9VdR^W^1zzYXM$6NcL)sbfD6b+iY5x=9qjMFIpcp=ZDbuKMeLiK8+ds8c2> zu#ihHJXm#XxY>Y2683&rj-~T-?}f2f{6g74YqOB`#CxCF%tgta9G!V9P*^Np;#OZe z&{kvI6}k7bp9`O%q4Q$`i}DF@>3J793|l*`^*0SFcoX7F=*^QYn-T5XZoKJkxW{%n`wQKfg7 zHcy&_bO{5RN&&G8h|p{P4=Kf^_T|78T3jDUALmUVLU15~NviCJnNG1f+AvM$K_Ca62xl zpYHxV_d)P*8+e7$-dYI8w^(R*iMc9?z1kl9I4_aU(P>S>*x(qDjhf-v+;~R5KCCk_ z_Q7eH1`{Qk|GI10PFFHp;l7ZBN{|gh{QZMG;1ITDJv*G-5AC>|{#Z_Km0nZF^^WhB zX#LZHyBwhud31yjRlw%j*slDc`TMoV^F4CQ$>EePH(@8AO&ovl=68%FRvVF%OO}fe z#l<|rGs(FPk}p59p0%S_4o{D;(I(4@yFiG~H%C6~IFmkgLctSthK!guEMGY^uT+4; zwCvBqjVLi-N#e2;#Yu$W7L)=BaS;lJ$fs9X%vDCW{Fe`tT21v-Av83Y7#MO9^IAZo zBFF1jZ!vzD93SFl7L@F- zPlO6LQHR)POOS+wEGM;@Y~r$hVp+Jkj{u=@F~55p<}eefbjQi8erYzug+_XF!a9Rn|Fn-6LLj9zZ3aE zeAADZI$0;VE_WtKEMp^02Xf;C7fr(RtwdH|c$sa$F9Y=o0p-(`mV@j3i@Z_vtF8fg z?&*BI3%@GSj~3!Te@KDT{l5F|oIIY$z|{RqTT!lVPnI{uyBviuo62Y`|-4 zdhJ_{V{p5qXA^HpY(XGt>Lg^;M|$;!HowwSRX_dD_AngGM%)q+{Sr zx=l^@wh_0DCIP!TxG}hgJZdFi3aEufbi7UTL-_O^^moN`Z=Mf(Og;soYj+zSk-us1 z#44?>w3{7$z48q>9?vEgh2bnCg;5#JB#d;+n0ssx!2(oA;em-vS?dI(tpie%>2wb+ zZgtog4mB^d)*baS{4V--Vm;Jdq|Ow+fG5zYe&ZNTBpUZ z9dK`aiN{tf_7XsgOs|5bF@DfX@Dk5qCC@I_rGF^Cwwwgsn_)MId)Ao08hUDNU|8w0 zyCW+_D>HMPi(1?D?^aZ9s~^hkYi3O7f4IB7;C2#2*thOP z@Ux22iDYM9;FEw%g9Ufh%mz(hS^4Js7O}tQn6SIav@;2VLIUBAaa*BSgm8VPXB0@{ zqomAmU3*}LuS<=lXsoD5P%1?e5B<8@=)D^kb)w%P zveU+uMatR54Sxh45`ix zwGVnKz#*)G8D(Z3UE|l_;h7fJfqM-b=cwo5Pa81*-69&x>9*=ZwkIk*XL;E9FwblF z#v@phi7{M?xYGKBI{vlPr*imt0))nN$#mP1=rDIHTMu1Xww;o>Gib|usRZpD%8|EW2K<1GxM$?mmuwyLF&3OG?oE!?U{^(@pLt*^Oejh z#eV5kkmXIf^d~a#((bqjj;`P;H`yGsS!KC4Hi6(3+NkGVj#7&4H^Ek5C|Ko@4y>90 zr@cA2q#oub4&fIO>leF_Rd$FQ}GGq?7ma&zAYWh7RcUl zy1ur9T72Hgv9};venOpYR1!!DTLO>}A4Vs+vb?_|TK1EZ9GfwHB8UH!%FMO zxWktzR6k|W;@Z1ur$3F=lA}(wo>icWfVv{xWzx+;A5ZzClOO!V{U-)z!8!$NVR6Z4 z^A)dArd$a|@U;}=m9SI}p2`?}k2`R8J`TC5BHZnJCq)UVN+Fjt9PBhG)p)LUTs?#v z$KOtjLdTC+?3q+sC^883lv7ESXrjVqMt0(e0U}Yl@SjegfK0S)l(}WSH%VrteCw*$ zy^;94<5`2jr$pq;J}F;3!GXr6NQL}3Jt(?#DTR7Io4>bXidQ2DQ*0#0uFw`3g)hc2 zYmwMxitdu4q?0$pX3U%^-|oiCZ*=VOOTFev-Gflhruz&N`Oi`Fp{%PQuKz~3FdO8B zVrN^;d1Hjrg-5(6FHxQUY0P&4XO3bm_E;3RhZJ3zEa;acj-3$Z3 zUoke7E}WZZp>(z^cpyH<`);+!YKaT?5|%1wpgH}XI10fl_N0cha+blPI&brJay@FB zJ6EQ_FR42jtQ4S)C|J~P&eQ(LUCdW&288!AaqVCX7^^Tb};j z{VM`ho3<$zyj4?C$PS`nVx zlmh);Hg{0CzL(!1f&4|ZQ7RWsJFp*Ph`0`Yp;5Dz1lXUG2o}rGjM;DXW5-5Yd9RJ? zsfFUsH&p6iMAe1pMDHpHKcP)QTN?gbsEJN<)t+lxlg2pJgqBwcSZBa`=j}#z7~k2C zavF?(d>Y;R1M(in$@D8RbT<@Xt$I5S)kNerX`S{ZpF&UPt?)TR|3_Z3yEWT4*IP{5 zPN$`f;Dh4#qhOqfJ5T@#^O{7r>zk-fk_8rg(|b-**`~as8N2oteFO$0+(4hDVch@# z86`E>s<3-De90h&Dzxcx-kgTR02rj3nu-RTr!D^=MV?GEbLfLn^^lcKUh0qnn$nL! z<4lwFGzw?8qJ(JZz!!x zo$ykxAKWxrm0`CPypleBkL*yZM{NhhMLh(p7}Lxda9^dw9)7?&@R@r>-=u|?JNHBx zU>#@q(S8P|{v+JDH74y`pSx4Qn z)~YIFaq;9&qo+i90Owlsonw^o~?+h z6TFxklq>IS5BG^KyW3#n9G80$lOOh5=UNwOwqBWT< z*A+QW&lVi35l_3E>ERK}CAOq&CB1HP^=B|mL6{KPNOfInNQ3X%n}?n-DL4XT_#GS~ zJ}PrknVE3b0Dtt?5`KYA>Sd7t`ikoLY9~_>45gHg$Ljr z6*1@h%Y}9#>Tqy`ITjCEN%aQ=p~kO4Hi;QlVZBOURC6N;C#@y#&2?_!{YHz2yb^Pz zl6I%#eYm~qky4~gYe&5$@TEAGru|->kMBJv3sIksj*qUdzxEsGwMhKX12cKq&o`RdSqWOQ$UO;(?u_MV?k;A{JX zyUhA}OzU5QpBHbB`*EbqH~XG(Tg_mNAp^a?b7T(JuS5O%_LotJEXK}uuqf`fpLaCY zLOXlO?ya27-Eb=cOdOSG;-8f4*6lyuJd%jzR{>n-I(d_u0mF=h{8M4NAmR` z2m8)kO$@GrK74GG?I!Y1a6JMQ-%pqdDMKSlTz^^Yq~nMT#U(832$h2OoEbg@pW<~D z(*^FU!euA6bF z)S@{?hxK;;q$51$V3j;i>DFAleQ3U1IcXo1na<}CM=Do7_|cqiUq86&EFTE3rARha zuNM+mFkVFUj71PoeRc_krt#z0vTLN*R`~GqW)wYIebxQR@kBZXh>B_Jf0Kpt53wAnsxhyZFvw{_N;oiJY9w!% z%=sj3hJE@WZpV%vRG@S9mq4pAXwH0BD<`cs&GLK#Yrdk(q8m7;kOF~8*6{roHf-(0 z#jdOUtw{yZIXKP~SF%Z%Wa8L6K?QT+um(dzBg5G~WW@n)caQDVJ*mwibSD!)T<~QB z_m772ZI7}k$qmM#&1YgR7MKgcqLWM!6uEg!hFmhLN+DJB(WIur8y<+rAQt(8?7>PX zw||;t>45v-$P3=CS*JzBFRs4$)vOWB#Z|9%AN$ZRommQcxZpzaJNa56T11S4CVf>( z)h{+Nn{K~S_Qb!f1ya4a^?G5+2P|TLm#yLuzMLY_6Wx{lR&-Sv5ydtD4(bAc+sQ0sd-oL0ipPb)5W_K;KMMV~Aqs%xkMsxprS}14;U%1qCQbs8|1ncR`sYmL-!hSq|EvT4 ObzdO>0Cp08-u?@OTiqN0 diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator/LICENSE.txt b/tests/llm_translation/test_skills_data/slack-gif-creator/LICENSE.txt deleted file mode 100644 index 7a4a3ea2424..00000000000 --- a/tests/llm_translation/test_skills_data/slack-gif-creator/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator/SKILL.md b/tests/llm_translation/test_skills_data/slack-gif-creator/SKILL.md index 16660d8ceb7..3cae971b731 100644 --- a/tests/llm_translation/test_skills_data/slack-gif-creator/SKILL.md +++ b/tests/llm_translation/test_skills_data/slack-gif-creator/SKILL.md @@ -1,7 +1,6 @@ --- name: slack-gif-creator description: Knowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." -license: Complete terms in LICENSE.txt --- # Slack GIF Creator From eea083fa4bab12a5cbc8e9f85b4821295791eadd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 27 Feb 2026 20:06:07 -0800 Subject: [PATCH 113/122] fix(mcp): default available_on_public_internet to true (#22331) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(mcp): default available_on_public_internet to true MCPs were defaulting to private (available_on_public_internet=false) which was a breaking change. This reverts the default to public (true) across: - Pydantic models (AddMCPServerRequest, UpdateMCPServerRequest, LiteLLM_MCPServerTable) - Prisma schema @default - mcp_server_manager.py YAML config + DB loading fallbacks - UI form initialValue and setFieldValue defaults * fix(ui): add forceRender to Collapse.Panel so toggle defaults render correctly Ant Design's Collapse.Panel lazy-renders children by default. Without forceRender, the Form.Item for 'Available on Public Internet' isn't mounted when the useEffect fires form.setFieldValue, causing the Switch to visually show OFF even though the intended default is true. Co-authored-by: Ishaan Jaff * fix(mcp): update remaining schema copies and MCPServer type default to true Missed in previous commit per Greptile review: - schema.prisma (root) - litellm-proxy-extras/litellm_proxy_extras/schema.prisma - litellm/types/mcp_server/mcp_server_manager.py MCPServer class * ui(mcp): reframe network access as 'Internal network only' restriction Replace scary 'Available on Public Internet' toggle with 'Internal network only' opt-in restriction. Toggle OFF (default) = all networks allowed. Toggle ON = restricted to internal network only. Auth is always required either way. - MCPPermissionManagement: new label/tooltip/description, invert display via getValueProps/getValueFromEvent so underlying available_on_public_internet value is unchanged - mcp_server_view: 'Public' → 'All networks', 'Internal' → 'Internal only' (orange) - mcp_server_columns: same badge updates --------- Co-authored-by: Cursor Agent Co-authored-by: Ishaan Jaff --- .../litellm_proxy_extras/schema.prisma | 2 +- ...odel_prices_and_context_window_backup.json | 55 +++++++++++++++++++ .../mcp_server/mcp_server_manager.py | 4 +- litellm/proxy/_types.py | 6 +- litellm/proxy/schema.prisma | 2 +- .../types/mcp_server/mcp_server_manager.py | 2 +- schema.prisma | 2 +- .../mcp_tools/MCPPermissionManagement.tsx | 13 +++-- .../mcp_tools/mcp_server_columns.tsx | 4 +- .../components/mcp_tools/mcp_server_view.tsx | 12 ++-- 10 files changed, 80 insertions(+), 22 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 13461be3e7c..2717480c7ef 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -300,7 +300,7 @@ model LiteLLM_MCPServerTable { token_url String? registration_url String? allow_all_keys Boolean @default(false) - available_on_public_internet Boolean @default(false) + available_on_public_internet Boolean @default(true) } // Generate Tokens for Proxy diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1ac8a347775..f52288ea72a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25157,6 +25157,25 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "openrouter/anthropic/claude-opus-4.6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346 + }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 3.75e-06, @@ -26169,6 +26188,42 @@ "supports_prompt_caching": true, "supports_computer_use": false }, + "openrouter/openrouter/auto": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "openrouter", + "max_input_tokens": 2000000, + "max_tokens": 2000000, + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_audio_input": true, + "supports_video_input": true + }, + "openrouter/openrouter/free": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "openrouter", + "max_input_tokens": 200000, + "max_tokens": 200000, + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "openrouter/openrouter/bodybuilder": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "openrouter", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "chat" + }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 7484de33ce4..08213f40b43 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -331,7 +331,7 @@ class MCPServerManager: static_headers=server_config.get("static_headers", None), allow_all_keys=bool(server_config.get("allow_all_keys", False)), available_on_public_internet=bool( - server_config.get("available_on_public_internet", False) + server_config.get("available_on_public_internet", True) ), ) self.config_mcp_servers[server_id] = new_server @@ -634,7 +634,7 @@ class MCPServerManager: disallowed_tools=getattr(mcp_server, "disallowed_tools", None), allow_all_keys=mcp_server.allow_all_keys, available_on_public_internet=bool( - getattr(mcp_server, "available_on_public_internet", False) + getattr(mcp_server, "available_on_public_internet", True) ), updated_at=getattr(mcp_server, "updated_at", None), ) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d2b39c73be1..afedb6c8e72 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1093,7 +1093,7 @@ class NewMCPServerRequest(LiteLLMPydanticObjectBase): token_url: Optional[str] = None registration_url: Optional[str] = None allow_all_keys: bool = False - available_on_public_internet: bool = False + available_on_public_internet: bool = True @model_validator(mode="before") @classmethod @@ -1147,7 +1147,7 @@ class UpdateMCPServerRequest(LiteLLMPydanticObjectBase): token_url: Optional[str] = None registration_url: Optional[str] = None allow_all_keys: bool = False - available_on_public_internet: bool = False + available_on_public_internet: bool = True @model_validator(mode="before") @classmethod @@ -1204,7 +1204,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase): token_url: Optional[str] = None registration_url: Optional[str] = None allow_all_keys: bool = False - available_on_public_internet: bool = False + available_on_public_internet: bool = True class MakeMCPServersPublicRequest(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index a5b0d930f58..f18556ac329 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -300,7 +300,7 @@ model LiteLLM_MCPServerTable { token_url String? registration_url String? allow_all_keys Boolean @default(false) - available_on_public_internet Boolean @default(false) + available_on_public_internet Boolean @default(true) } // Generate Tokens for Proxy diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 7f99fd526c8..69b34a25a21 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -52,7 +52,7 @@ class MCPServer(BaseModel): env: Optional[Dict[str, str]] = None access_groups: Optional[List[str]] = None allow_all_keys: bool = False - available_on_public_internet: bool = False + available_on_public_internet: bool = True updated_at: Optional[datetime] = None model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/schema.prisma b/schema.prisma index bc32a8cce32..a8cb297a3ed 100644 --- a/schema.prisma +++ b/schema.prisma @@ -300,7 +300,7 @@ model LiteLLM_MCPServerTable { token_url String? registration_url String? allow_all_keys Boolean @default(false) - available_on_public_internet Boolean @default(false) + available_on_public_internet Boolean @default(true) } // Generate Tokens for Proxy diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPPermissionManagement.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPPermissionManagement.tsx index 2adfe52f6f8..89aa756cb27 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPPermissionManagement.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPPermissionManagement.tsx @@ -46,7 +46,7 @@ const MCPPermissionManagement: React.FC = ({ } } else { form.setFieldValue("allow_all_keys", false); - form.setFieldValue("available_on_public_internet", false); + form.setFieldValue("available_on_public_internet", true); } }, [mcpServer, form]); @@ -64,6 +64,7 @@ const MCPPermissionManagement: React.FC = ({ } key="permissions" className="border-0" + forceRender >
@@ -89,17 +90,19 @@ const MCPPermissionManagement: React.FC = ({
- Available on Public Internet - + Internal network only + -

Enable if this server should be reachable from the public internet.

+

Turn on to restrict access to callers within your internal network only.

({ checked: !value })} + getValueFromEvent={(checked: boolean) => !checked} + initialValue={true} className="mb-0" > diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx index d49949e7e04..c42a8593cf1 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx @@ -157,9 +157,9 @@ export const mcpServerColumns = ( cell: ({ row }) => { const isPublic = row.original.available_on_public_internet; return isPublic ? ( - Public + All networks ) : ( - Internal + Internal only ); }, }, diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_view.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_view.tsx index cb7768ab21a..4f8ba4e1307 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_view.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_view.tsx @@ -251,20 +251,20 @@ export const MCPServerView: React.FC = ({
- Available on Public Internet + Network Access
{mcpServer.available_on_public_internet ? ( - Public + All networks ) : ( - - Internal + + Internal only )} - {mcpServer.available_on_public_internet && ( + {!mcpServer.available_on_public_internet && ( - Accessible from external/public IPs + Restricted to internal network )}
From ee703cea99252c0e45b2a74b9f376e2ecda9ea65 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 27 Feb 2026 20:30:47 -0800 Subject: [PATCH 114/122] fix(jwt): OIDC discovery URLs, roles array handling, dot-notation error hints (#22336) * fix(jwt): support OIDC discovery URLs, handle roles array, improve error hints Three fixes for Azure AD JWT auth: 1. OIDC discovery URL support - JWT_PUBLIC_KEY_URL can now be set to .well-known/openid-configuration endpoints. The proxy fetches the discovery doc, extracts jwks_uri, and caches it. 2. Handle roles claim as array - when team_id_jwt_field points to a list (e.g. AAD's "roles": ["team1"]), auto-unwrap the first element instead of crashing with 'unhashable type: list'. 3. Better error hint for dot-notation indexing - when team_id_jwt_field is set to "roles.0" or "roles[0]", the 401 error now explains to use "roles" instead and that LiteLLM auto-unwraps lists. * Add integration demo script for JWT auth fixes (OIDC discovery, array roles, dot-notation hints) Co-authored-by: Ishaan Jaff * Add demo_servers.py for manual JWT auth testing with mock JWKS/OIDC endpoints Co-authored-by: Ishaan Jaff * Add demo screenshots for PR comment Co-authored-by: Ishaan Jaff * Add integration test results with screenshots for PR review Co-authored-by: Ishaan Jaff * address greptile review feedback (greploop iteration 1) - fix: add HTTP status code check in _resolve_jwks_url before parsing JSON - fix: remove misleading bracket-notation hint from debug log (get_nested_value does not support it) * Update tests/test_litellm/proxy/auth/test_handle_jwt.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * remove demo scripts and assets --------- Co-authored-by: Cursor Agent Co-authored-by: Ishaan Jaff Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/proxy/auth/handle_jwt.py | 84 +++++- .../proxy/auth/test_handle_jwt.py | 269 ++++++++++++++++++ 2 files changed, 351 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 9921b74b561..553ba4d6c49 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -8,6 +8,7 @@ JWT token must have 'litellm_proxy_admin' in scope. import fnmatch import os +import re from typing import Any, List, Literal, Optional, Set, Tuple, cast from cryptography import x509 @@ -235,7 +236,17 @@ class JWTHandler: return self.litellm_jwtauth.team_id_default else: return default_value - # At this point, team_id is not the sentinel, so it should be a string + # AAD and other IdPs often send roles/groups as a list of strings. + # team_id_jwt_field is singular, so take the first element when a list + # is returned. This avoids "unhashable type: 'list'" errors downstream. + if isinstance(team_id, list): + if not team_id: + return default_value + verbose_proxy_logger.debug( + f"JWT Auth: team_id_jwt_field '{self.litellm_jwtauth.team_id_jwt_field}' " + f"returned a list {team_id}; using first element '{team_id[0]}' automatically." + ) + team_id = team_id[0] return team_id # type: ignore[return-value] elif self.litellm_jwtauth.team_id_default is not None: team_id = self.litellm_jwtauth.team_id_default @@ -453,6 +464,52 @@ class JWTHandler: scopes = [] return scopes + async def _resolve_jwks_url(self, url: str) -> str: + """ + If url points to an OIDC discovery document (*.well-known/openid-configuration), + fetch it and return the jwks_uri contained within. Otherwise return url unchanged. + This lets JWT_PUBLIC_KEY_URL be set to a well-known discovery endpoint instead of + requiring operators to manually find the JWKS URL. + """ + if ".well-known/openid-configuration" not in url: + return url + + cache_key = f"litellm_oidc_discovery_{url}" + cached_jwks_uri = await self.user_api_key_cache.async_get_cache(cache_key) + if cached_jwks_uri is not None: + return cached_jwks_uri + + verbose_proxy_logger.debug( + f"JWT Auth: Fetching OIDC discovery document from {url}" + ) + response = await self.http_handler.get(url) + if response.status_code != 200: + raise Exception( + f"JWT Auth: OIDC discovery endpoint {url} returned status {response.status_code}: {response.text}" + ) + try: + discovery = response.json() + except Exception as e: + raise Exception( + f"JWT Auth: Failed to parse OIDC discovery document at {url}: {e}" + ) + + jwks_uri = discovery.get("jwks_uri") + if not jwks_uri: + raise Exception( + f"JWT Auth: OIDC discovery document at {url} does not contain a 'jwks_uri' field." + ) + + verbose_proxy_logger.debug( + f"JWT Auth: Resolved OIDC discovery {url} -> jwks_uri={jwks_uri}" + ) + await self.user_api_key_cache.async_set_cache( + key=cache_key, + value=jwks_uri, + ttl=self.litellm_jwtauth.public_key_ttl, + ) + return jwks_uri + async def get_public_key(self, kid: Optional[str]) -> dict: keys_url = os.getenv("JWT_PUBLIC_KEY_URL") @@ -462,6 +519,7 @@ class JWTHandler: keys_url_list = [url.strip() for url in keys_url.split(",")] for key_url in keys_url_list: + key_url = await self._resolve_jwks_url(key_url) cache_key = f"litellm_jwt_auth_keys_{key_url}" cached_keys = await self.user_api_key_cache.async_get_cache(cache_key) @@ -913,8 +971,30 @@ class JWTAuthManager: if jwt_handler.is_required_team_id() is True: team_id_field = jwt_handler.litellm_jwtauth.team_id_jwt_field team_alias_field = jwt_handler.litellm_jwtauth.team_alias_jwt_field + hint = "" + if team_id_field: + # "roles.0" — dot-notation numeric indexing is not supported + if "." in team_id_field: + parts = team_id_field.rsplit(".", 1) + if parts[-1].isdigit(): + base_field = parts[0] + hint = ( + f" Hint: dot-notation array indexing (e.g. '{team_id_field}') is not " + f"supported. Use '{base_field}' instead — LiteLLM automatically " + f"uses the first element when the field value is a list." + ) + # "roles[0]" — bracket-notation indexing is also not supported in get_nested_value + elif "[" in team_id_field and team_id_field.endswith("]"): + m = re.match(r"^(\w+)\[(\d+)\]$", team_id_field) + if m: + base_field = m.group(1) + hint = ( + f" Hint: array indexing (e.g. '{team_id_field}') is not supported " + f"in team_id_jwt_field. Use '{base_field}' instead — LiteLLM " + f"automatically uses the first element when the field value is a list." + ) raise Exception( - f"No team found in token. Checked team_id field '{team_id_field}' and team_alias field '{team_alias_field}'" + f"No team found in token. Checked team_id field '{team_id_field}' and team_alias field '{team_alias_field}'.{hint}" ) return individual_team_id, team_object diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index b56d13bb932..8418dde5e9c 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -1485,4 +1485,273 @@ async def test_get_objects_resolves_org_by_name(): ) +# --------------------------------------------------------------------------- +# Fix 1: OIDC discovery URL resolution +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_resolve_jwks_url_passthrough_for_direct_jwks_url(): + """Non-discovery URLs are returned unchanged.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.caching.dual_cache import DualCache + + handler = JWTHandler() + handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + url = "https://login.microsoftonline.com/common/discovery/keys" + result = await handler._resolve_jwks_url(url) + assert result == url + + +@pytest.mark.asyncio +async def test_resolve_jwks_url_resolves_oidc_discovery_document(): + """ + A .well-known/openid-configuration URL should be fetched and its + jwks_uri returned. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.caching.dual_cache import DualCache + + handler = JWTHandler() + cache = DualCache() + handler.update_environment( + prisma_client=None, + user_api_key_cache=cache, + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + + discovery_url = "https://login.microsoftonline.com/tenant/.well-known/openid-configuration" + jwks_url = "https://login.microsoftonline.com/tenant/discovery/keys" + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"jwks_uri": jwks_url, "issuer": "https://..."} + + with patch.object(handler.http_handler, "get", new_callable=AsyncMock, return_value=mock_response) as mock_get: + result = await handler._resolve_jwks_url(discovery_url) + + assert result == jwks_url + mock_get.assert_called_once_with(discovery_url) + + +@pytest.mark.asyncio +async def test_resolve_jwks_url_caches_resolved_jwks_uri(): + """Resolved jwks_uri is cached — second call does not hit the network.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.caching.dual_cache import DualCache + + handler = JWTHandler() + cache = DualCache() + handler.update_environment( + prisma_client=None, + user_api_key_cache=cache, + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + + discovery_url = "https://login.microsoftonline.com/tenant/.well-known/openid-configuration" + jwks_url = "https://login.microsoftonline.com/tenant/discovery/keys" + + mock_response = MagicMock() + mock_response.json.return_value = {"jwks_uri": jwks_url} + + with patch.object(handler.http_handler, "get", new_callable=AsyncMock, return_value=mock_response) as mock_get: + first = await handler._resolve_jwks_url(discovery_url) + second = await handler._resolve_jwks_url(discovery_url) + + assert first == jwks_url + assert second == jwks_url + # Network should only be hit once + assert mock_get.call_count == 1 + + +@pytest.mark.asyncio +async def test_resolve_jwks_url_raises_if_no_jwks_uri_in_discovery_doc(): + """Raise a helpful error if the discovery document has no jwks_uri.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.caching.dual_cache import DualCache + + handler = JWTHandler() + handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + + discovery_url = "https://example.com/.well-known/openid-configuration" + mock_response = MagicMock() + mock_response.json.return_value = {"issuer": "https://example.com"} # no jwks_uri + + with patch.object(handler.http_handler, "get", new_callable=AsyncMock, return_value=mock_response): + with pytest.raises(Exception, match="jwks_uri"): + await handler._resolve_jwks_url(discovery_url) + + +# --------------------------------------------------------------------------- +# Fix 2: handle array values in team_id_jwt_field (e.g. AAD "roles" claim) +# --------------------------------------------------------------------------- + + +def _make_jwt_handler(team_id_jwt_field: str) -> JWTHandler: + from litellm.caching.dual_cache import DualCache + + handler = JWTHandler() + handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth(team_id_jwt_field=team_id_jwt_field), + ) + return handler + + +def test_get_team_id_returns_first_element_when_roles_is_list(): + """ + AAD sends roles as a list. get_team_id() must return the first string + element rather than the raw list (which would later crash with + 'unhashable type: list'). + """ + handler = _make_jwt_handler("roles") + token = {"oid": "user-oid", "roles": ["team1"]} + result = handler.get_team_id(token=token, default_value=None) + assert result == "team1" + + +def test_get_team_id_returns_first_element_from_multi_value_roles_list(): + """When roles has multiple entries, the first one is used.""" + handler = _make_jwt_handler("roles") + token = {"roles": ["team2", "team1"]} + result = handler.get_team_id(token=token, default_value=None) + assert result == "team2" + + +def test_get_team_id_returns_default_when_roles_list_is_empty(): + """Empty list should fall back to default_value.""" + handler = _make_jwt_handler("roles") + token = {"roles": []} + result = handler.get_team_id(token=token, default_value="fallback") + assert result == "fallback" + + +def test_get_team_id_still_works_with_string_value(): + """String values (non-array) continue to work as before.""" + handler = _make_jwt_handler("appid") + token = {"appid": "my-team-id"} + result = handler.get_team_id(token=token, default_value=None) + assert result == "my-team-id" + + +def test_get_team_id_list_result_is_hashable(): + """ + The value returned by get_team_id() must be hashable so it can be + added to a set (the operation that previously crashed). + """ + handler = _make_jwt_handler("roles") + token = {"roles": ["team1"]} + result = handler.get_team_id(token=token, default_value=None) + # This must not raise TypeError + s: set = set() + s.add(result) + assert "team1" in s + + +# --------------------------------------------------------------------------- +# Fix 3: helpful error message for dot-notation array indexing (roles.0) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_find_and_validate_specific_team_id_hints_bracket_notation(): + """ + When team_id_jwt_field is set to 'roles.0' (unsupported dot-notation for + array indexing) and no team is found, the exception message should suggest + using 'roles' instead (and explain LiteLLM auto-unwraps list values). + """ + from unittest.mock import MagicMock + + from litellm.caching.dual_cache import DualCache + + handler = _make_jwt_handler("roles.0") + # token has roles as a list — dot-notation won't find anything + token = {"roles": ["team1"]} + + with pytest.raises(Exception) as exc_info: + await JWTAuthManager.find_and_validate_specific_team_id( + jwt_handler=handler, + jwt_valid_token=token, + prisma_client=None, + user_api_key_cache=DualCache(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + error_msg = str(exc_info.value) + # Should mention the bad field name and suggest the fix + assert "roles.0" in error_msg, f"Expected field name in: {error_msg}" + assert "roles" in error_msg and "list" in error_msg, ( + f"Expected hint about using 'roles' instead: {error_msg}" + ) + + +@pytest.mark.asyncio +async def test_find_and_validate_specific_team_id_hints_bracket_index_notation(): + """ + When team_id_jwt_field is set to 'roles[0]' (bracket indexing, also unsupported + in get_nested_value) the error message should suggest using 'roles' instead. + """ + from unittest.mock import MagicMock + + from litellm.caching.dual_cache import DualCache + + handler = _make_jwt_handler("roles[0]") + token = {"roles": ["team1"]} + + with pytest.raises(Exception) as exc_info: + await JWTAuthManager.find_and_validate_specific_team_id( + jwt_handler=handler, + jwt_valid_token=token, + prisma_client=None, + user_api_key_cache=DualCache(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + error_msg = str(exc_info.value) + assert "roles[0]" in error_msg, f"Expected field name in: {error_msg}" + assert "roles" in error_msg and "list" in error_msg, ( + f"Expected hint about using 'roles' instead: {error_msg}" + ) + + +@pytest.mark.asyncio +async def test_find_and_validate_specific_team_id_no_hint_for_valid_field(): + """ + When team_id_jwt_field is a normal field name (no dot-notation) the + error message should not contain a spurious bracket-notation hint. + """ + from unittest.mock import AsyncMock, MagicMock + + from litellm.caching.dual_cache import DualCache + + handler = _make_jwt_handler("appid") + token = {} # no appid — triggers the "no team found" path + + with pytest.raises(Exception) as exc_info: + await JWTAuthManager.find_and_validate_specific_team_id( + jwt_handler=handler, + jwt_valid_token=token, + prisma_client=None, + user_api_key_cache=DualCache(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + error_msg = str(exc_info.value) + assert "Hint" not in error_msg From 9f2453712ec054b8ad8888d314bb24f6c42316c6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 27 Feb 2026 20:45:53 -0800 Subject: [PATCH 115/122] =?UTF-8?q?perf:=20streaming=20latency=20improveme?= =?UTF-8?q?nts=20=E2=80=94=204=20targeted=20hot-path=20fixes=20(#22346)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: raise aiohttp connection pool limits (300→1000, 50/host→500) * perf: skip model_copy() on every chunk — only copy usage-bearing chunks * perf: replace list+join O(n²) with str+= O(n) in async_data_generator * perf: cache model-level guardrail lookup per request, not per chunk --- litellm/constants.py | 4 +- .../litellm_core_utils/streaming_handler.py | 21 ++- litellm/proxy/proxy_server.py | 10 +- litellm/proxy/utils.py | 149 +++++++++++------- 4 files changed, 112 insertions(+), 72 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 3d2cebf2224..4c38ecd74b5 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -193,9 +193,9 @@ _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client fo # Aiohttp connection pooling - prevents memory leaks from unbounded connection growth # Set to 0 for unlimited (not recommended for production) -AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 300)) +AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000)) AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int( - os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 50) + os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500) ) AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120)) AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300)) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index baf274f2c62..3b75a56fcc9 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1968,22 +1968,24 @@ class CustomStreamWrapper: self.rules.post_call_rules( input=self.response_uptil_now, model=self.model ) - # Store a shallow copy so usage stripping below - # does not mutate the stored chunk. - self.chunks.append(processed_chunk.model_copy()) - # Add mcp_list_tools to first chunk if present if not self.sent_first_chunk: processed_chunk = self._add_mcp_list_tools_to_first_chunk(processed_chunk) self.sent_first_chunk = True - if ( + + _has_usage = ( hasattr(processed_chunk, "usage") and getattr(processed_chunk, "usage", None) is not None - ): + ) + + if _has_usage: + # Store a copy ONLY when usage stripping below will mutate + # the chunk. For non-usage chunks (vast majority), store + # directly to avoid expensive model_copy() per chunk. + self.chunks.append(processed_chunk.model_copy()) + # Strip usage from the outgoing chunk so it's not sent twice # (once in the chunk, once in _hidden_params). - # Create a new object without usage, matching sync behavior. - # The copy in self.chunks retains usage for calculate_total_usage(). obj_dict = processed_chunk.model_dump() if "usage" in obj_dict: del obj_dict["usage"] @@ -1995,6 +1997,9 @@ class CustomStreamWrapper: ) if is_empty: continue + else: + # No usage data — safe to store directly without copying + self.chunks.append(processed_chunk) # add usage as hidden param if self.sent_last_chunk is True and self.stream_options is None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 48025863641..bd5b5309e0f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5302,13 +5302,15 @@ async def async_data_generator( ): verbose_proxy_logger.debug("inside generator") try: - # Use a list to accumulate response segments to avoid O(n^2) string concatenation - str_so_far_parts: list[str] = [] error_message: Optional[str] = None requested_model_from_client = _get_client_requested_model_for_streaming( request_data=request_data ) model_mismatch_logged = False + # Use a running string instead of list + join to avoid O(n^2) overhead. + # Previously "".join(str_so_far_parts) was called every chunk, re-joining + # the entire accumulated response. String += is O(n) amortized total. + _str_so_far: str = "" async for chunk in proxy_logging_obj.async_post_call_streaming_iterator_hook( user_api_key_dict=user_api_key_dict, response=response, @@ -5319,12 +5321,12 @@ async def async_data_generator( user_api_key_dict=user_api_key_dict, response=chunk, data=request_data, - str_so_far="".join(str_so_far_parts), + str_so_far=_str_so_far if _str_so_far else None, ) if isinstance(chunk, (ModelResponse, ModelResponseStream)): response_str = litellm.get_response_string(response_obj=chunk) - str_so_far_parts.append(response_str) + _str_so_far += response_str chunk, model_mismatch_logged = _restamp_streaming_chunk_model( chunk=chunk, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index f6613b5548f..5e0d5336aa9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -23,23 +23,31 @@ from typing import ( ) from litellm import _custom_logger_compatible_callbacks_literal -from litellm.constants import (DEFAULT_MODEL_CREATED_AT_TIME, - MAX_TEAM_LIST_LIMIT) -from litellm.proxy._types import (DB_CONNECTION_ERROR_TYPES, CommonProxyErrors, - ProxyErrorTypes, ProxyException, - SpendLogsMetadata, SpendLogsPayload) +from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME, MAX_TEAM_LIST_LIMIT +from litellm.proxy._types import ( + DB_CONNECTION_ERROR_TYPES, + CommonProxyErrors, + ProxyErrorTypes, + ProxyException, + SpendLogsMetadata, + SpendLogsPayload, +) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import CallTypes, CallTypesLiteral try: - from litellm_enterprise.enterprise_callbacks.send_emails.base_email import \ - BaseEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import \ - ResendEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import \ - SendGridEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import \ - SMTPEmailLogger + from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( + BaseEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( + ResendEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import ( + SendGridEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import ( + SMTPEmailLogger, + ) except ImportError: BaseEmailLogger = None # type: ignore SendGridEmailLogger = None # type: ignore @@ -58,56 +66,70 @@ from fastapi import HTTPException, status import litellm import litellm.litellm_core_utils import litellm.litellm_core_utils.litellm_logging -from litellm import (EmbeddingResponse, ImageResponse, ModelResponse, - ModelResponseStream, Router) +from litellm import ( + EmbeddingResponse, + ImageResponse, + ModelResponse, + ModelResponseStream, + Router, +) from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.caching.caching import DualCache, RedisCache from litellm.caching.dual_cache import LimitedSizeOrderedDict from litellm.exceptions import RejectedRequestError -from litellm.integrations.custom_guardrail import (CustomGuardrail, - ModifyResponseException) +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + ModifyResponseException, +) from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting -from litellm.integrations.SlackAlerting.utils import \ - _add_langfuse_trace_id_to_alert +from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.proxy._types import (AlertType, CallInfo, - LiteLLM_VerificationTokenView, Member, - UserAPIKeyAuth) +from litellm.proxy._types import ( + AlertType, + CallInfo, + LiteLLM_VerificationTokenView, + Member, + UserAPIKeyAuth, +) from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.db.create_views import (create_missing_views, - should_create_missing_views) +from litellm.proxy.db.create_views import ( + create_missing_views, + should_create_missing_views, +) from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.db.log_db_metrics import log_db_metrics from litellm.proxy.db.prisma_client import PrismaWrapper -from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import \ - UnifiedLLMGuardrails +from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( + UnifiedLLMGuardrails, +) from litellm.proxy.hooks import PROXY_HOOKS, get_proxy_hook from litellm.proxy.hooks.cache_control_check import _PROXY_CacheControlCheck from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter -from litellm.proxy.hooks.parallel_request_limiter import \ - _PROXY_MaxParallelRequestsHandler +from litellm.proxy.hooks.parallel_request_limiter import ( + _PROXY_MaxParallelRequestsHandler, +) from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES -from litellm.types.mcp import (MCPDuringCallResponseObject, - MCPPreCallRequestObject, - MCPPreCallResponseObject) -from litellm.types.proxy.policy_engine.pipeline_types import \ - PipelineExecutionResult +from litellm.types.mcp import ( + MCPDuringCallResponseObject, + MCPPreCallRequestObject, + MCPPreCallResponseObject, +) +from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionResult from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - from litellm.litellm_core_utils.litellm_logging import \ - Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj Span = Union[_Span, Any] else: @@ -1050,9 +1072,10 @@ class ProxyLogging: """Process prompt template if applicable.""" from litellm.proxy.prompts.prompt_endpoints import ( - construct_versioned_prompt_id, get_latest_version_prompt_id) - from litellm.proxy.prompts.prompt_registry import \ - IN_MEMORY_PROMPT_REGISTRY + construct_versioned_prompt_id, + get_latest_version_prompt_id, + ) + from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY from litellm.utils import get_non_default_completion_params if prompt_version is None: @@ -1102,8 +1125,9 @@ class ProxyLogging: def _process_guardrail_metadata(self, data: dict) -> None: """Process guardrails from metadata and add to applied_guardrails.""" - from litellm.proxy.common_utils.callback_utils import \ - add_guardrail_to_applied_guardrails_header + from litellm.proxy.common_utils.callback_utils import ( + add_guardrail_to_applied_guardrails_header, + ) metadata_standard = data.get("metadata") or {} metadata_litellm = data.get("litellm_metadata") or {} @@ -2000,27 +2024,32 @@ class ProxyLogging: if isinstance(response, (ModelResponse, ModelResponseStream)): response_str = litellm.get_response_string(response_obj=response) elif isinstance(response, dict) and self.is_a2a_streaming_response(response): - from litellm.llms.a2a.common_utils import \ - extract_text_from_a2a_response + from litellm.llms.a2a.common_utils import extract_text_from_a2a_response response_str = extract_text_from_a2a_response(response) if response_str is not None: + # Cache model-level guardrails check per-request to avoid repeated + # dict lookups + llm_router.get_deployment() per callback per chunk. + _cached_guardrail_data: Optional[dict] = None + _guardrail_data_computed = False + for callback in litellm.callbacks: try: _callback: Optional[CustomLogger] = None if isinstance(callback, CustomGuardrail): # Main - V2 Guardrails implementation - from litellm.types.guardrails import \ - GuardrailEventHooks + from litellm.types.guardrails import GuardrailEventHooks - ## CHECK FOR MODEL-LEVEL GUARDRAILS - modified_data = _check_and_merge_model_level_guardrails( - data=data, llm_router=llm_router - ) + ## CHECK FOR MODEL-LEVEL GUARDRAILS (cached per-request) + if not _guardrail_data_computed: + _cached_guardrail_data = _check_and_merge_model_level_guardrails( + data=data, llm_router=llm_router + ) + _guardrail_data_computed = True if ( callback.should_run_guardrail( - data=modified_data, + data=_cached_guardrail_data, event_type=GuardrailEventHooks.post_call, ) is not True @@ -4626,8 +4655,9 @@ async def update_spend_logs_job( # Guardrail/policy usage tracking (same batch, outside spend-logs update) try: - from litellm.proxy.guardrails.usage_tracking import \ - process_spend_logs_guardrail_usage + from litellm.proxy.guardrails.usage_tracking import ( + process_spend_logs_guardrail_usage, + ) await process_spend_logs_guardrail_usage( prisma_client=prisma_client, logs_to_process=logs_to_process, @@ -4653,8 +4683,10 @@ async def _monitor_spend_logs_queue( db_writer_client: Optional HTTP handler for external spend logs endpoint proxy_logging_obj: Proxy logging object """ - from litellm.constants import (SPEND_LOG_QUEUE_POLL_INTERVAL, - SPEND_LOG_QUEUE_SIZE_THRESHOLD) + from litellm.constants import ( + SPEND_LOG_QUEUE_POLL_INTERVAL, + SPEND_LOG_QUEUE_SIZE_THRESHOLD, + ) threshold = SPEND_LOG_QUEUE_SIZE_THRESHOLD base_interval = SPEND_LOG_QUEUE_POLL_INTERVAL @@ -5175,11 +5207,12 @@ async def get_available_models_for_user( List of model names available to the user """ from litellm.proxy.auth.auth_checks import get_team_object - from litellm.proxy.auth.model_checks import (get_complete_model_list, - get_key_models, - get_team_models) - from litellm.proxy.management_endpoints.team_endpoints import \ - validate_membership + from litellm.proxy.auth.model_checks import ( + get_complete_model_list, + get_key_models, + get_team_models, + ) + from litellm.proxy.management_endpoints.team_endpoints import validate_membership # Get proxy model list and access groups if llm_router is None: From c4a0174e0072218b00574053f952ac2c4e33db69 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 27 Feb 2026 21:16:41 -0800 Subject: [PATCH 116/122] test: add comprehensive Vitest coverage for CostTrackingSettings Add 88 tests across 9 test files for the CostTrackingSettings component directory: - provider_display_helpers.test.ts: 9 tests for helper functions - how_it_works.test.tsx: 9 tests for discount calculator component - add_provider_form.test.tsx: 7 tests for provider form validation - add_margin_form.test.tsx: 9 tests for margin form with type toggle - provider_discount_table.test.tsx: 12 tests for table editing and interactions - provider_margin_table.test.tsx: 13 tests for margin table with sorting - use_discount_config.test.ts: 11 tests for discount hook logic - use_margin_config.test.ts: 12 tests for margin hook logic - cost_tracking_settings.test.tsx: 15 tests for main component and role-based rendering All tests passing. Coverage includes form validation, user interactions, API calls, state management, and conditional rendering. Co-Authored-By: Claude Haiku 4.5 --- .../add_margin_form.test.tsx | 148 ++++++++ .../add_provider_form.test.tsx | 98 ++++++ .../cost_tracking_settings.test.tsx | 201 +++++++++++ .../how_it_works.test.tsx | 95 ++++++ .../provider_discount_table.test.tsx | 241 +++++++++++++ .../provider_display_helpers.test.ts | 91 +++++ .../provider_margin_table.test.tsx | 246 ++++++++++++++ .../use_discount_config.test.ts | 244 ++++++++++++++ .../use_margin_config.test.ts | 316 ++++++++++++++++++ 9 files changed, 1680 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/add_margin_form.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/add_provider_form.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/how_it_works.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/provider_discount_table.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/provider_display_helpers.test.ts create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/provider_margin_table.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/use_discount_config.test.ts create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/use_margin_config.test.ts diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/add_margin_form.test.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/add_margin_form.test.tsx new file mode 100644 index 00000000000..d61f0987ac2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/add_margin_form.test.tsx @@ -0,0 +1,148 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../tests/test-utils"; +import AddMarginForm from "./add_margin_form"; +import { MarginConfig } from "./types"; + +vi.mock("../provider_info_helpers", () => ({ + Providers: { + OpenAI: "OpenAI", + Anthropic: "Anthropic", + }, + provider_map: { + OpenAI: "openai", + Anthropic: "anthropic", + }, + providerLogoMap: { + OpenAI: "https://example.com/openai.png", + Anthropic: "https://example.com/anthropic.png", + }, +})); + +vi.mock("./provider_display_helpers", () => ({ + handleImageError: vi.fn(), +})); + +const DEFAULT_PROPS = { + marginConfig: {} as MarginConfig, + selectedProvider: undefined, + marginType: "percentage" as const, + percentageValue: "", + fixedAmountValue: "", + onProviderChange: vi.fn(), + onMarginTypeChange: vi.fn(), + onPercentageChange: vi.fn(), + onFixedAmountChange: vi.fn(), + onAddProvider: vi.fn(), +}; + +describe("AddMarginForm", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /add provider margin/i })).toBeInTheDocument(); + }); + + it("should show the percentage input when marginType is percentage", () => { + renderWithProviders(); + expect(screen.getByPlaceholderText("10")).toBeInTheDocument(); + }); + + it("should show the fixed amount input when marginType is fixed", () => { + renderWithProviders(); + expect(screen.getByPlaceholderText("0.001")).toBeInTheDocument(); + }); + + it("should not show the fixed amount input when marginType is percentage", () => { + renderWithProviders(); + expect(screen.queryByPlaceholderText("0.001")).not.toBeInTheDocument(); + }); + + it("should not show the percentage input when marginType is fixed", () => { + renderWithProviders(); + expect(screen.queryByPlaceholderText("10")).not.toBeInTheDocument(); + }); + + it("should show the Percentage-based and Fixed Amount radio options", () => { + renderWithProviders(); + expect(screen.getByText("Percentage-based")).toBeInTheDocument(); + expect(screen.getByText("Fixed Amount")).toBeInTheDocument(); + }); + + it("should disable the submit button when no provider is selected (percentage mode)", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider margin/i })).toBeDisabled(); + }); + + it("should disable the submit button when provider is selected but no percentage value (percentage mode)", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider margin/i })).toBeDisabled(); + }); + + it("should enable the submit button when provider and percentage value are both provided", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider margin/i })).not.toBeDisabled(); + }); + + it("should disable the submit button in fixed mode when no fixed amount is provided", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider margin/i })).toBeDisabled(); + }); + + it("should enable the submit button in fixed mode when provider and fixed amount are provided", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider margin/i })).not.toBeDisabled(); + }); + + it("should call onAddProvider when the enabled submit button is clicked", async () => { + const onAddProvider = vi.fn(); + const user = userEvent.setup(); + renderWithProviders( + + ); + + await user.click(screen.getByRole("button", { name: /add provider margin/i })); + expect(onAddProvider).toHaveBeenCalledTimes(1); + }); + + it("should call onMarginTypeChange when the Fixed Amount radio is clicked", async () => { + const onMarginTypeChange = vi.fn(); + const user = userEvent.setup(); + renderWithProviders( + + ); + + await user.click(screen.getByText("Fixed Amount")); + expect(onMarginTypeChange).toHaveBeenCalledWith("fixed"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/add_provider_form.test.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/add_provider_form.test.tsx new file mode 100644 index 00000000000..611c8609c36 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/add_provider_form.test.tsx @@ -0,0 +1,98 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../tests/test-utils"; +import AddProviderForm from "./add_provider_form"; +import { DiscountConfig } from "./types"; + +vi.mock("../provider_info_helpers", () => ({ + Providers: { + OpenAI: "OpenAI", + Anthropic: "Anthropic", + }, + provider_map: { + OpenAI: "openai", + Anthropic: "anthropic", + }, + providerLogoMap: { + OpenAI: "https://example.com/openai.png", + Anthropic: "https://example.com/anthropic.png", + }, +})); + +vi.mock("./provider_display_helpers", () => ({ + handleImageError: vi.fn(), +})); + +const DEFAULT_PROPS = { + discountConfig: {} as DiscountConfig, + selectedProvider: undefined, + newDiscount: "", + onProviderChange: vi.fn(), + onDiscountChange: vi.fn(), + onAddProvider: vi.fn(), +}; + +describe("AddProviderForm", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /add provider discount/i })).toBeInTheDocument(); + }); + + it("should render the discount percentage input field", () => { + renderWithProviders(); + expect(screen.getByPlaceholderText("5")).toBeInTheDocument(); + }); + + it("should disable the submit button when no provider is selected and no discount is entered", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /add provider discount/i })).toBeDisabled(); + }); + + it("should disable the submit button when a provider is selected but no discount is entered", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider discount/i })).toBeDisabled(); + }); + + it("should disable the submit button when a discount is entered but no provider is selected", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider discount/i })).toBeDisabled(); + }); + + it("should enable the submit button when both a provider and a discount value are provided", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider discount/i })).not.toBeDisabled(); + }); + + it("should call onAddProvider when the enabled submit button is clicked", async () => { + const onAddProvider = vi.fn(); + const user = userEvent.setup(); + renderWithProviders( + + ); + + await user.click(screen.getByRole("button", { name: /add provider discount/i })); + expect(onAddProvider).toHaveBeenCalledTimes(1); + }); + + it("should show the percent sign next to the discount input", () => { + renderWithProviders(); + expect(screen.getByText("%")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.test.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.test.tsx new file mode 100644 index 00000000000..db6899ba17f --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.test.tsx @@ -0,0 +1,201 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../tests/test-utils"; +import CostTrackingSettings from "./cost_tracking_settings"; + +// Mock sub-hooks so we can control their state without network calls +const mockDiscountConfig = vi.fn(() => ({})); +const mockMarginConfig = vi.fn(() => ({})); + +vi.mock("./use_discount_config", () => ({ + useDiscountConfig: () => ({ + discountConfig: mockDiscountConfig(), + fetchDiscountConfig: vi.fn().mockResolvedValue(undefined), + handleAddProvider: vi.fn().mockResolvedValue(true), + handleRemoveProvider: vi.fn().mockResolvedValue(undefined), + handleDiscountChange: vi.fn().mockResolvedValue(undefined), + }), +})); + +vi.mock("./use_margin_config", () => ({ + useMarginConfig: () => ({ + marginConfig: mockMarginConfig(), + fetchMarginConfig: vi.fn().mockResolvedValue(undefined), + handleAddMargin: vi.fn().mockResolvedValue(true), + handleRemoveMargin: vi.fn().mockResolvedValue(undefined), + handleMarginChange: vi.fn().mockResolvedValue(undefined), + }), +})); + +vi.mock("./pricing_calculator/index", () => ({ + default: () =>
Pricing Calculator
, +})); + +vi.mock("../playground/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn().mockResolvedValue([]), +})); + +vi.mock("../HelpLink", () => ({ + DocsMenu: () => null, +})); + +vi.mock("./how_it_works", () => ({ + default: () =>
How It Works
, +})); + +vi.mock("../provider_info_helpers", () => ({ + Providers: { OpenAI: "OpenAI" }, + provider_map: { OpenAI: "openai" }, + providerLogoMap: {}, +})); + +vi.mock("./provider_display_helpers", () => ({ + getProviderDisplayInfo: vi.fn(() => ({ displayName: "OpenAI", logo: "", enumKey: "OpenAI" })), + handleImageError: vi.fn(), +})); + +const ADMIN_PROPS = { + userID: "user-1", + userRole: "proxy_admin", + accessToken: "test-token", +}; + +describe("CostTrackingSettings", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockDiscountConfig.mockReturnValue({}); + mockMarginConfig.mockReturnValue({}); + }); + + it("should return nothing when accessToken is null", () => { + const { container } = renderWithProviders( + + ); + expect(container.firstChild).toBeNull(); + }); + + it("should render the page title", () => { + renderWithProviders(); + expect(screen.getByText("Cost Tracking Settings")).toBeInTheDocument(); + }); + + it("should show the Provider Discounts accordion header for proxy_admin", () => { + renderWithProviders(); + expect(screen.getByText("Provider Discounts")).toBeInTheDocument(); + }); + + it("should show the Fee/Price Margin accordion header for proxy_admin", () => { + renderWithProviders(); + expect(screen.getByText("Fee/Price Margin")).toBeInTheDocument(); + }); + + it("should always show the Pricing Calculator section", () => { + renderWithProviders(); + // The accordion header text appears in the DOM; getAllByText tolerates duplicates + expect(screen.getAllByText("Pricing Calculator").length).toBeGreaterThan(0); + }); + + it("should show the pricing calculator component", async () => { + renderWithProviders(); + expect(await screen.findByTestId("pricing-calculator")).toBeInTheDocument(); + }); + + it("should not show Provider Discounts section for a non-admin role", () => { + renderWithProviders( + + ); + expect(screen.queryByText("Provider Discounts")).not.toBeInTheDocument(); + }); + + it("should not show Fee/Price Margin section for a non-admin role", () => { + renderWithProviders( + + ); + expect(screen.queryByText("Fee/Price Margin")).not.toBeInTheDocument(); + }); + + it("should show Provider Discounts for the 'Admin' role as well", () => { + renderWithProviders( + + ); + expect(screen.getByText("Provider Discounts")).toBeInTheDocument(); + }); + + it("should show the subtitle describing discount/margin configuration", () => { + renderWithProviders(); + expect( + screen.getByText(/configure cost discounts and margins/i) + ).toBeInTheDocument(); + }); + + describe("Add Provider Discount modal", () => { + it("should open the Add Provider Discount modal when the button is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + // The button lives inside the Provider Discounts accordion — click the header to expand first + const accordionHeader = screen.getByText("Provider Discounts").closest("button"); + if (accordionHeader) { + await user.click(accordionHeader); + } + + const addButton = await screen.findByRole("button", { name: /add provider discount/i }); + await user.click(addButton); + + expect( + await screen.findByText("Add Provider Discount", { selector: "h2" }) + ).toBeInTheDocument(); + }); + }); + + describe("Add Provider Margin modal", () => { + it("should open the Add Provider Margin modal when the button is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const accordionHeader = screen.getByText("Fee/Price Margin").closest("button"); + if (accordionHeader) { + await user.click(accordionHeader); + } + + const addButton = await screen.findByRole("button", { name: /add provider margin/i }); + await user.click(addButton); + + expect( + await screen.findByText("Add Provider Margin", { selector: "h2" }) + ).toBeInTheDocument(); + }); + }); + + describe("empty state messages", () => { + it("should show the empty state message when no discount config is loaded", async () => { + mockDiscountConfig.mockReturnValue({}); + renderWithProviders(); + + const accordionHeader = screen.getByText("Provider Discounts").closest("button"); + if (accordionHeader) { + await userEvent.setup().click(accordionHeader); + } + + expect( + await screen.findByText(/no provider discounts configured/i) + ).toBeInTheDocument(); + }); + + it("should show the empty state message when no margin config is loaded", async () => { + mockMarginConfig.mockReturnValue({}); + renderWithProviders(); + + const accordionHeader = screen.getByText("Fee/Price Margin").closest("button"); + if (accordionHeader) { + await userEvent.setup().click(accordionHeader); + } + + expect( + await screen.findByText(/no provider margins configured/i) + ).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/how_it_works.test.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/how_it_works.test.tsx new file mode 100644 index 00000000000..fa608f555ce --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/how_it_works.test.tsx @@ -0,0 +1,95 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../tests/test-utils"; +import HowItWorks from "./how_it_works"; + +vi.mock("@/app/(dashboard)/api-reference/components/CodeBlock", () => ({ + default: ({ code }: { code: string }) => 
{code}
, +})); + +describe("HowItWorks", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render", () => { + renderWithProviders(); + expect(screen.getByText("Cost Calculation")).toBeInTheDocument(); + }); + + it("should display the cost calculation formula", () => { + renderWithProviders(); + expect(screen.getByText(/final_cost = base_cost/i)).toBeInTheDocument(); + }); + + it("should display the valid range information", () => { + renderWithProviders(); + expect(screen.getByText(/0% and 100%/i)).toBeInTheDocument(); + }); + + it("should render the code block with a curl example", () => { + renderWithProviders(); + expect(screen.getByTestId("code-block")).toBeInTheDocument(); + expect(screen.getByTestId("code-block").textContent).toContain("curl"); + }); + + it("should show the response header names for discount verification", () => { + renderWithProviders(); + expect(screen.getByText("x-litellm-response-cost")).toBeInTheDocument(); + expect(screen.getByText("x-litellm-response-cost-original")).toBeInTheDocument(); + expect(screen.getByText("x-litellm-response-cost-discount-amount")).toBeInTheDocument(); + }); + + it("should not show calculated results initially when no input is provided", () => { + renderWithProviders(); + expect(screen.queryByText("Calculated Results")).not.toBeInTheDocument(); + }); + + it("should not show calculated results when only response cost is entered", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const responseCostInput = screen.getByPlaceholderText("0.0171938125"); + await user.type(responseCostInput, "0.01"); + + expect(screen.queryByText("Calculated Results")).not.toBeInTheDocument(); + }); + + it("should not show calculated results when only discount amount is entered", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const discountAmountInput = screen.getByPlaceholderText("0.0009049375"); + await user.type(discountAmountInput, "0.001"); + + expect(screen.queryByText("Calculated Results")).not.toBeInTheDocument(); + }); + + it("should show calculated results when both fields are filled", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const responseCostInput = screen.getByPlaceholderText("0.0171938125"); + const discountAmountInput = screen.getByPlaceholderText("0.0009049375"); + + await user.type(responseCostInput, "0.0171938125"); + await user.type(discountAmountInput, "0.0009049375"); + + expect(await screen.findByText("Calculated Results")).toBeInTheDocument(); + }); + + it("should show original cost, final cost, and discount amount in results", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.type(screen.getByPlaceholderText("0.0171938125"), "0.0171938125"); + await user.type(screen.getByPlaceholderText("0.0009049375"), "0.0009049375"); + + expect(await screen.findByText("Original Cost:")).toBeInTheDocument(); + expect(screen.getByText("Final Cost:")).toBeInTheDocument(); + expect(screen.getByText("Discount Amount:")).toBeInTheDocument(); + expect(screen.getByText("Discount Applied:")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/provider_discount_table.test.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/provider_discount_table.test.tsx new file mode 100644 index 00000000000..7697c6e7686 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/provider_discount_table.test.tsx @@ -0,0 +1,241 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../tests/test-utils"; +import ProviderDiscountTable from "./provider_discount_table"; + +vi.mock("@heroicons/react/outline", () => ({ + TrashIcon: function TrashIcon() { return null; }, + PencilAltIcon: function PencilAltIcon() { return null; }, + CheckIcon: function CheckIcon() { return null; }, + XIcon: function XIcon() { return null; }, +})); + +vi.mock("@tremor/react", () => ({ + Table: ({ children }: any) =>
{children}
, + TableHead: ({ children }: any) => {children}, + TableRow: ({ children }: any) => {children}, + TableHeaderCell: ({ children }: any) => {children}, + TableBody: ({ children }: any) => {children}, + TableCell: ({ children }: any) => {children}, + Text: ({ children }: any) => {children}, + TextInput: ({ value, onValueChange, onKeyDown, placeholder, ...rest }: any) => ( + onValueChange?.(e.target.value)} + onKeyDown={onKeyDown} + placeholder={placeholder} + {...rest} + /> + ), + Icon: ({ icon: IconComponent, onClick }: any) => { + const name = IconComponent?.displayName ?? IconComponent?.name ?? "icon"; + return + + + {/* Project Details */} + + + + {project.description || "\u2014"} + + {new Date(project.created_at).toLocaleString()} + {project.created_by && ( + +  {"by"}  + + + )} + + + {new Date(project.updated_at).toLocaleString()} + {project.updated_by && ( + +  {"by"}  + + + )} + + + + + + {/* Spend / Budget */} + + + + + Budget + + } + style={{ height: "100%" }} + > + +
+ + ${spend.toFixed(2)} + +
+ {hasLimit ? `of $${maxBudget.toFixed(2)} budget` : "No budget limit"} +
+ {hasLimit && ( +
+ + + {(Math.round(spendPercent * 10) / 10).toFixed(1)}% utilized + +
+ )} + + + + + + {modelSpendData.length > 0 ? ( + `$${value.toFixed(4)}`} + yAxisWidth={140} + showLegend={false} + style={{ height: Math.max(modelSpendData.length * 40, 120) }} + /> + ) : ( + + )} + + + + + {/* Keys & Team */} + + + + + Keys + + } + style={{ height: "100%" }} + > + + + + + + + Team + + } + style={{ height: "100%" }} + > + {teamInfo ? ( + (() => { + const teamBudget = teamInfo.max_budget ?? null; + const teamSpend = teamInfo.spend ?? 0; + const teamHasLimit = teamBudget != null && teamBudget > 0; + const teamPercent = teamHasLimit ? Math.min((teamSpend / teamBudget) * 100, 100) : 0; + const teamColor = teamPercent >= 90 ? "#f5222d" : teamPercent >= 70 ? "#faad14" : "#52c41a"; + + return ( + + {/* Team name + ID */} +
+ + {teamInfo.team_alias || teamInfo.team_id} + +
+ + ID:{" "} + + {teamInfo.team_id} + + +
+ + {/* Models */} +
+ + Models + + {(teamInfo.models?.length ?? 0) > 0 ? ( + + {teamInfo.models?.map((m: string) => ( + + {m} + + ))} + + ) : ( + All models + )} +
+ + {/* Budget + Spend compact */} +
+ + + Spend + + + ${teamSpend.toFixed(2)} + {teamHasLimit ? ( + + {" "} + / ${teamBudget.toFixed(2)} + + ) : ( + + {" "} + (Unlimited) + + )} + + + {teamHasLimit && ( + + )} +
+ + {/* Members */} + + + Members + + {teamInfo.members_with_roles?.length ?? 0} + + + ); + })() + ) : project.team_id ? ( + + } size="small" /> + + ) : ( + + )} + + + + + {/* Edit Modal */} + setIsEditModalVisible(false)} /> + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx index 14b4d70b743..4010699360b 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx @@ -1,95 +1,39 @@ -import { useEffect, useState } from "react"; +import { Modal, Form, Button, Typography, message } from "antd"; +import { FolderAddOutlined } from "@ant-design/icons"; import { - Alert, - Modal, - Form, - Input, - Select, - Switch, - InputNumber, - Collapse, - Button, - Col, - Flex, - Row, - Space, - Divider, - Typography, - message, -} from "antd"; -import { FolderAddOutlined, PlusOutlined, MinusCircleOutlined } from "@ant-design/icons"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; -import { useCreateProject, ProjectCreateParams } from "@/app/(dashboard)/hooks/projects/useCreateProject"; -import { Team } from "../../key_team_helpers/key_list"; -import { fetchTeamModels } from "../../organisms/create_key_button"; -import { getModelDisplayName } from "../../key_team_helpers/fetch_available_models_team_key"; + useCreateProject, + ProjectCreateParams, +} from "@/app/(dashboard)/hooks/projects/useCreateProject"; +import { + ProjectBaseForm, + ProjectFormValues, +} from "./ProjectBaseForm"; +import { buildProjectApiParams } from "./projectFormUtils"; interface CreateProjectModalProps { isOpen: boolean; onClose: () => void; } -export function CreateProjectModal({ isOpen, onClose }: CreateProjectModalProps) { - const [form] = Form.useForm(); - const { accessToken, userId, userRole } = useAuthorized(); - const { data: teams } = useTeams(); +export function CreateProjectModal({ + isOpen, + onClose, +}: CreateProjectModalProps) { + const [form] = Form.useForm(); const createMutation = useCreateProject(); - const [selectedTeam, setSelectedTeam] = useState(null); - const [modelsToPick, setModelsToPick] = useState([]); - - // Fetch team-scoped models when team selection changes - useEffect(() => { - if (userId && userRole && accessToken && selectedTeam) { - fetchTeamModels(userId, userRole, accessToken, selectedTeam.team_id).then((models) => { - const allModels = Array.from(new Set([...(selectedTeam.models ?? []), ...models])); - setModelsToPick(allModels); - }); - } else { - setModelsToPick([]); - } - form.setFieldValue("models", []); - }, [selectedTeam, accessToken, userId, userRole, form]); - const handleSubmit = async () => { try { const values = await form.validateFields(); - - // Build model-specific limits from the dynamic form list - const modelRpmLimit: Record = {}; - const modelTpmLimit: Record = {}; - for (const entry of values.modelLimits ?? []) { - if (entry.model) { - if (entry.rpm != null) modelRpmLimit[entry.model] = entry.rpm; - if (entry.tpm != null) modelTpmLimit[entry.model] = entry.tpm; - } - } - - // Build metadata from the dynamic form list - const metadata: Record = {}; - for (const entry of values.metadata ?? []) { - if (entry.key) metadata[entry.key] = entry.value; - } - const params: ProjectCreateParams = { - project_alias: values.project_alias, - description: values.description, + ...buildProjectApiParams(values), team_id: values.team_id, - models: values.models ?? [], - max_budget: values.max_budget, - blocked: values.isBlocked ?? false, - ...(Object.keys(modelRpmLimit).length > 0 && { model_rpm_limit: modelRpmLimit }), - ...(Object.keys(modelTpmLimit).length > 0 && { model_tpm_limit: modelTpmLimit }), - ...(Object.keys(metadata).length > 0 && { metadata }), }; createMutation.mutate(params, { onSuccess: () => { message.success("Project created successfully"); form.resetFields(); - setSelectedTeam(null); - setModelsToPick([]); onClose(); }, onError: (error) => { @@ -103,16 +47,9 @@ export function CreateProjectModal({ isOpen, onClose }: CreateProjectModalProps) const handleCancel = () => { form.resetFields(); - setSelectedTeam(null); - setModelsToPick([]); onClose(); }; - const handleTeamChange = (teamId: string) => { - const team = teams?.find((t) => t.team_id === teamId) ?? null; - setSelectedTeam(team); - }; - return ( Cancel , - , ]} > -
- {/* Basic Info */} - - Basic Information - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {/* Advanced Settings */} - - - - - Advanced Settings - - } - key="1" - > - - Block Project - - - - - prev.isBlocked !== cur.isBlocked}> - {({ getFieldValue }) => - getFieldValue("isBlocked") ? ( - - ) : null - } - - - - - - Model-Specific Limits - - - {(fields, { add, remove }) => ( - <> - {fields.map(({ key, name, ...restField }) => ( - - - - - - - - - - - remove(name)} style={{ color: "#ef4444" }} /> - - ))} - - - - - )} - - - - - - Metadata - - - {(fields, { add, remove }) => ( - <> - {fields.map(({ key, name, ...restField }) => ( - - - - - - - - remove(name)} style={{ color: "#ef4444" }} /> - - ))} - - - - - )} - - - - - - +
); } diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx new file mode 100644 index 00000000000..75f56b1373f --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx @@ -0,0 +1,126 @@ +import { useEffect } from "react"; +import { Modal, Form, Button, Typography, message } from "antd"; +import { SaveOutlined } from "@ant-design/icons"; +import { ProjectResponse } from "@/app/(dashboard)/hooks/projects/useProjects"; +import { + useUpdateProject, + ProjectUpdateParams, +} from "@/app/(dashboard)/hooks/projects/useUpdateProject"; +import { ProjectBaseForm, ProjectFormValues } from "./ProjectBaseForm"; +import { buildProjectApiParams } from "./projectFormUtils"; + +interface EditProjectModalProps { + isOpen: boolean; + project: ProjectResponse; + onClose: () => void; + onSuccess?: () => void; +} + +export function EditProjectModal({ + isOpen, + project, + onClose, + onSuccess, +}: EditProjectModalProps) { + const [form] = Form.useForm(); + const updateMutation = useUpdateProject(); + + // Populate form with existing project data when modal opens + useEffect(() => { + if (isOpen && project) { + // Model limits are stored inside metadata by the backend + const metadataObj = (project.metadata ?? {}) as Record; + const rpmLimits = (metadataObj.model_rpm_limit ?? {}) as Record; + const tpmLimits = (metadataObj.model_tpm_limit ?? {}) as Record; + + const modelLimits: ProjectFormValues["modelLimits"] = []; + const allLimitModels = new Set([ + ...Object.keys(rpmLimits), + ...Object.keys(tpmLimits), + ]); + for (const model of allLimitModels) { + modelLimits.push({ + model, + rpm: rpmLimits[model], + tpm: tpmLimits[model], + }); + } + + // Filter out internal keys from user-facing metadata + const internalKeys = new Set(["model_rpm_limit", "model_tpm_limit"]); + const metadata: ProjectFormValues["metadata"] = []; + for (const [key, value] of Object.entries(metadataObj)) { + if (!internalKeys.has(key)) { + metadata.push({ key, value: String(value) }); + } + } + + form.setFieldsValue({ + project_alias: project.project_alias ?? "", + team_id: project.team_id ?? "", + description: project.description ?? "", + models: project.models ?? [], + max_budget: project.litellm_budget_table?.max_budget ?? undefined, + isBlocked: project.blocked, + modelLimits: modelLimits.length > 0 ? modelLimits : undefined, + metadata: metadata.length > 0 ? metadata : undefined, + }); + } + }, [isOpen, project, form]); + + const handleSubmit = async () => { + try { + const values = await form.validateFields(); + const params: ProjectUpdateParams = { + ...buildProjectApiParams(values), + team_id: values.team_id, + }; + + updateMutation.mutate( + { projectId: project.project_id, params }, + { + onSuccess: () => { + message.success("Project updated successfully"); + onSuccess?.(); + onClose(); + }, + onError: (error) => { + message.error(error.message || "Failed to update project"); + }, + }, + ); + } catch (error) { + console.error("Validation failed:", error); + } + }; + + return ( + + Edit Project + + } + open={isOpen} + onCancel={onClose} + width={720} + destroyOnHidden + footer={[ + , + , + ]} + > + + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx new file mode 100644 index 00000000000..bf1eca882c3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx @@ -0,0 +1,401 @@ +import { useEffect, useState } from "react"; +import { + Alert, + Col, + Collapse, + Divider, + Flex, + Form, + Input, + InputNumber, + Row, + Select, + Space, + Switch, + Typography, + Button, +} from "antd"; +import type { FormInstance } from "antd"; +import { PlusOutlined, MinusCircleOutlined } from "@ant-design/icons"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { Team } from "../../key_team_helpers/key_list"; +import { fetchTeamModels } from "../../organisms/create_key_button"; +import { getModelDisplayName } from "../../key_team_helpers/fetch_available_models_team_key"; + +export interface ProjectFormValues { + project_alias: string; + team_id: string; + description?: string; + models: string[]; + max_budget?: number; + isBlocked: boolean; + modelLimits?: { model: string; tpm?: number; rpm?: number }[]; + metadata?: { key: string; value: string }[]; +} + +interface ProjectBaseFormProps { + form: FormInstance; +} + +export function ProjectBaseForm({ + form, +}: ProjectBaseFormProps) { + const { accessToken, userId, userRole } = useAuthorized(); + const { data: teams } = useTeams(); + + const [selectedTeam, setSelectedTeam] = useState(null); + const [modelsToPick, setModelsToPick] = useState([]); + + // Sync selectedTeam from form value (needed for edit mode pre-fill) + const teamIdValue = Form.useWatch("team_id", form); + useEffect(() => { + if (teamIdValue && teams) { + const team = teams.find((t) => t.team_id === teamIdValue) ?? null; + if (team && team.team_id !== selectedTeam?.team_id) { + setSelectedTeam(team); + } + } + }, [teamIdValue, teams, selectedTeam?.team_id]); + + // Fetch team-scoped models when team selection changes + useEffect(() => { + if (userId && userRole && accessToken && selectedTeam) { + fetchTeamModels(userId, userRole, accessToken, selectedTeam.team_id).then( + (models) => { + const allModels = Array.from( + new Set([...(selectedTeam.models ?? []), ...models]), + ); + setModelsToPick(allModels); + }, + ); + } else { + setModelsToPick([]); + } + }, [selectedTeam, accessToken, userId, userRole]); + + const handleTeamChange = (teamId: string) => { + const team = teams?.find((t) => t.team_id === teamId) ?? null; + setSelectedTeam(team); + form.setFieldValue("models", []); + }; + + return ( +
+ {/* Basic Info */} + + Basic Information + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {/* Advanced Settings */} + + + + Advanced Settings + + ), + children: ( + <> + + Block Project + + + + + prev.isBlocked !== cur.isBlocked} + > + {({ getFieldValue }) => + getFieldValue("isBlocked") ? ( + + ) : null + } + + + + + + Model-Specific Limits + + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name, ...restField }) => ( + + { + if (!value) return Promise.resolve(); + const all = form.getFieldValue("modelLimits") ?? []; + const dupes = all.filter( + (entry: { model?: string }) => entry?.model === value, + ); + if (dupes.length > 1) { + return Promise.reject(new Error("Duplicate model")); + } + return Promise.resolve(); + }, + }, + ]} + > + + + + + + + + + remove(name)} + style={{ color: "#ef4444" }} + /> + + ))} + + + + + )} + + + + + + Metadata + + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name, ...restField }) => ( + + { + if (!value) return Promise.resolve(); + const all = form.getFieldValue("metadata") ?? []; + const dupes = all.filter( + (entry: { key?: string }) => entry?.key === value, + ); + if (dupes.length > 1) { + return Promise.reject(new Error("Duplicate key")); + } + return Promise.resolve(); + }, + }, + ]} + > + + + + + + remove(name)} + style={{ color: "#ef4444" }} + /> + + ))} + + + + + )} + + + ), + }, + ]} + /> + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/projectFormUtils.ts b/ui/litellm-dashboard/src/components/Projects/ProjectModals/projectFormUtils.ts new file mode 100644 index 00000000000..97c093b57d9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/projectFormUtils.ts @@ -0,0 +1,36 @@ +import { ProjectFormValues } from "./ProjectBaseForm"; + +/** + * Transforms ProjectFormValues into the flat API param shape + * shared by both create and update endpoints. + */ +export function buildProjectApiParams(values: ProjectFormValues) { + const modelRpmLimit: Record = {}; + const modelTpmLimit: Record = {}; + for (const entry of values.modelLimits ?? []) { + if (entry.model) { + if (entry.rpm != null) modelRpmLimit[entry.model] = entry.rpm; + if (entry.tpm != null) modelTpmLimit[entry.model] = entry.tpm; + } + } + + const metadata: Record = {}; + for (const entry of values.metadata ?? []) { + if (entry.key) metadata[entry.key] = entry.value; + } + + return { + project_alias: values.project_alias, + description: values.description, + models: values.models ?? [], + max_budget: values.max_budget, + blocked: values.isBlocked ?? false, + ...(Object.keys(modelRpmLimit).length > 0 && { + model_rpm_limit: modelRpmLimit, + }), + ...(Object.keys(modelTpmLimit).length > 0 && { + model_tpm_limit: modelTpmLimit, + }), + ...(Object.keys(metadata).length > 0 && { metadata }), + }; +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx index 40ab5045703..f0b593c2e49 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx @@ -1,13 +1,15 @@ import { useProjects, ProjectResponse } from "@/app/(dashboard)/hooks/projects/useProjects"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; -import { PlusOutlined } from "@ant-design/icons"; +import { LoadingOutlined, PlusOutlined } from "@ant-design/icons"; import { Button, Card, Flex, Input, Layout, + Pagination, Space, + Spin, Table, Tag, theme, @@ -18,6 +20,7 @@ import type { ColumnsType } from "antd/es/table"; import { LayersIcon, SearchIcon } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import { CreateProjectModal } from "./ProjectModals/CreateProjectModal"; +import { ProjectDetail } from "./ProjectDetailsPage"; const { Title, Text } = Typography; const { Content } = Layout; @@ -25,8 +28,9 @@ const { Content } = Layout; export function ProjectsPage() { const { token } = theme.useToken(); const { data: projects, isLoading } = useProjects(); - const { data: teams } = useTeams(); + const { data: teams, isLoading: isTeamsLoading } = useTeams(); + const [selectedProjectId, setSelectedProjectId] = useState(null); const [isCreateModalVisible, setIsCreateModalVisible] = useState(false); const [searchText, setSearchText] = useState(""); const [currentPage, setCurrentPage] = useState(1); @@ -74,6 +78,7 @@ export function ProjectsPage() { ellipsis className="text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs cursor-pointer" style={{ fontSize: 14, padding: "1px 8px" }} + onClick={() => setSelectedProjectId(id)} > {id} @@ -96,8 +101,11 @@ export function ProjectsPage() { return aAlias.localeCompare(bAlias); }, render: (_: unknown, record: ProjectResponse) => { - const alias = teamAliasMap.get(record.team_id ?? ""); - return alias ?? record.team_id ?? "—"; + if (!record.team_id) return "—"; + const alias = teamAliasMap.get(record.team_id); + if (alias) return alias; + if (isTeamsLoading) return } size="small" />; + return record.team_id; }, }, { @@ -144,6 +152,15 @@ export function ProjectsPage() { }, ]; + if (selectedProjectId) { + return ( + setSelectedProjectId(null)} + /> + ); + } + return ( setSearchText(e.target.value)} allowClear /> + setCurrentPage(page)} + size="small" + showTotal={(total) => `${total} projects`} + showSizeChanger={false} + /> setCurrentPage(page), - size: "small", - showTotal: (total) => `${total} projects`, - showSizeChanger: false, - }} + pagination={false} /> From 50dc7b520cc7ba0211af064598b69a435b9da661 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 27 Feb 2026 22:11:04 -0800 Subject: [PATCH 119/122] Update ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../components/Projects/ProjectModals/CreateProjectModal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx index 4010699360b..e490f89303f 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx @@ -60,7 +60,7 @@ export function CreateProjectModal({ open={isOpen} onCancel={handleCancel} width={720} - destroyOnClose + destroyOnHidden footer={[ - - - Keys - - } - style={{ height: "100%" }} - > - - + (""); + + const { data, isLoading } = useKeys(page, PAGE_SIZE, { + projectID: projectId, + selectedKeyAlias: keyAlias || null, + }); + + // Reset to page 1 when filter changes + useEffect(() => { + setPage(1); + }, [keyAlias]); + + const keys = data?.keys ?? []; + const totalCount = data?.total_count ?? 0; + + return ( + + + Keys + + } + style={{ height: "100%" }} + > + + } + placeholder="Filter by key name..." + style={{ maxWidth: 220 }} + value={keyAlias} + onChange={(e) => setKeyAlias(e.target.value)} + allowClear + size="small" + /> + `${total} keys`} + /> + + } /> } : false} + /> + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectKeysTable.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectKeysTable.tsx new file mode 100644 index 00000000000..cb80d0e27a5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectKeysTable.tsx @@ -0,0 +1,58 @@ +import { KeyResponse } from "@/components/key_team_helpers/key_list"; +import { Empty, Table, Tooltip } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import type { SpinProps } from "antd"; +import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; + +interface ProjectKeysTableProps { + keys: KeyResponse[]; + loading?: boolean | SpinProps; +} + +const columns: ColumnsType = [ + { + title: "Key Name", + dataIndex: "key_alias", + key: "key_alias", + render: (alias: string | null) => alias || "—", + }, + { + title: "Owner", + key: "owner", + render: (_: unknown, record: KeyResponse) => { + const email = record.user?.user_email ?? record.user_id ?? null; + if (!email) return "—"; + return ( + + + + ); + }, + }, + { + title: "Created", + dataIndex: "created_at", + key: "created_at", + render: (date: string) => (date ? new Date(date).toLocaleDateString() : "—"), + }, + { + title: "Last Active", + dataIndex: "last_active", + key: "last_active", + render: (date: string | null) => (date ? new Date(date).toLocaleDateString() : "Never"), + }, +]; + +export function ProjectKeysTable({ keys, loading }: ProjectKeysTableProps) { + return ( +
}} + /> + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx index f0b593c2e49..9c75e19ac4e 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx @@ -2,6 +2,7 @@ import { useProjects, ProjectResponse } from "@/app/(dashboard)/hooks/projects/u import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { LoadingOutlined, PlusOutlined } from "@ant-design/icons"; import { + Alert, Button, Card, Flex, @@ -165,6 +166,12 @@ export function ProjectsPage() { + void; + disabled?: boolean; + loading?: boolean; + /** When set, only show projects belonging to this team */ + teamId?: string | null; +} + +const ProjectDropdown: React.FC = ({ + projects, + value, + onChange, + disabled, + loading, + teamId, +}) => { + const filtered = teamId + ? projects?.filter((p) => p.team_id === teamId) + : projects; + + return ( + + ); +}; + +export default ProjectDropdown; diff --git a/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx b/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx index d54724da2a7..9e79ea2950a 100644 --- a/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx +++ b/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx @@ -7,10 +7,10 @@ interface TeamDropdownProps { value?: string; onChange?: (value: string) => void; disabled?: boolean; + loading?: boolean; } -const TeamDropdown: React.FC = ({ teams, value, onChange, disabled }) => { - console.log("disabled", disabled); +const TeamDropdown: React.FC = ({ teams, value, onChange, disabled, loading }) => { return ( onChange?.(e.target.value)} + /> + ), +})); + vi.mock("../common_components/AccessGroupSelector", () => ({ default: ({ value = [], onChange }: { value?: string[]; onChange?: (v: string[]) => void }) => ( = ({ team, teams, data, addKey }) => { const { accessToken, userId: userID, userRole, premiumUser } = useAuthorized(); + const { data: projects, isLoading: isProjectsLoading } = useProjects(); const queryClient = useQueryClient(); const [form] = Form.useForm(); const [isModalVisible, setIsModalVisible] = useState(false); @@ -157,6 +160,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { const [promptsList, setPromptsList] = useState([]); const [loggingSettings, setLoggingSettings] = useState([]); const [selectedCreateKeyTeam, setSelectedCreateKeyTeam] = useState(team); + const [selectedProjectId, setSelectedProjectId] = useState(null); const [isCreateUserModalVisible, setIsCreateUserModalVisible] = useState(false); const [newlyCreatedUserId, setNewlyCreatedUserId] = useState(null); const [possibleUIRoles, setPossibleUIRoles] = useState>>({}); @@ -184,6 +188,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { setRouterSettings(null); setRouterSettingsKey((prev) => prev + 1); setSelectedAgentId(null); + setSelectedProjectId(null); }; const handleCancel = () => { @@ -200,6 +205,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { setRouterSettings(null); setRouterSettingsKey((prev) => prev + 1); setSelectedAgentId(null); + setSelectedProjectId(null); }; useEffect(() => { @@ -468,6 +474,14 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { }; useEffect(() => { + if (selectedProjectId) { + // When a project is selected, use the project's models + const project = projects?.find((p) => p.project_id === selectedProjectId); + const projectModels = project?.models ?? []; + setModelsToPick(projectModels); + form.setFieldValue("models", []); + return; + } if (userID && userRole && accessToken) { fetchTeamModels(userID, userRole, accessToken, selectedCreateKeyTeam?.team_id ?? null).then((models) => { let allModels = Array.from(new Set([...(selectedCreateKeyTeam?.models ?? []), ...models])); @@ -475,7 +489,22 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { }); } form.setFieldValue("models", []); - }, [selectedCreateKeyTeam, accessToken, userID, userRole]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedCreateKeyTeam, selectedProjectId, accessToken, userID, userRole]); + + // Sync team when project is selected but teams loaded later (race condition) + useEffect(() => { + if (!selectedProjectId || !teams) return; + const project = projects?.find((p) => p.project_id === selectedProjectId); + if (!project?.team_id) return; + // If team is already set correctly, skip + if (selectedCreateKeyTeam?.team_id === project.team_id) return; + const projectTeam = teams.find((t) => t.team_id === project.team_id) || null; + if (projectTeam) { + setSelectedCreateKeyTeam(projectTeam); + form.setFieldValue("team_id", projectTeam.team_id); + } + }, [teams, selectedProjectId, projects]); // Add a callback function to handle user creation const handleUserCreated = (userId: string) => { @@ -653,9 +682,40 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { > { const selectedTeam = teams?.find((t) => t.team_id === teamId) || null; setSelectedCreateKeyTeam(selectedTeam); + setSelectedProjectId(null); + form.setFieldValue("project_id", undefined); + }} + /> + + + Project{" "} + + + + + } + name="project_id" + className="mt-4" + > + { + if (!projectId) { + setSelectedProjectId(null); + setSelectedCreateKeyTeam(null); + form.setFieldValue("team_id", undefined); + return; + } + setSelectedProjectId(projectId); }} /> @@ -735,9 +795,11 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { } }} > - + {!selectedProjectId && ( + + )} {modelsToPick.map((model: string) => (