From 562afb208d586cdb944a0052dec430a12f2618af Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 4 Dec 2025 12:30:08 -0800 Subject: [PATCH 01/40] v0 customer usage, pending tests + extras version bump --- .../litellm_proxy_extras/schema.prisma | 28 ++++ litellm/constants.py | 1 + litellm/proxy/_types.py | 2 + litellm/proxy/db/db_spend_update_writer.py | 120 +++++++++++++++++- .../redis_update_buffer.py | 36 ++++++ .../customer_endpoints.py | 80 +++++++++++- litellm/proxy/schema.prisma | 28 ++++ schema.prisma | 28 ++++ 8 files changed, 319 insertions(+), 4 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 2883dfc4b82..4d4a127f7e8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -462,6 +462,34 @@ model LiteLLM_DailyOrganizationSpend { @@index([mcp_namespaced_tool_name]) } +// Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyEndUserSpend { + id String @id @default(uuid()) + end_user_id String? + date String + api_key String + model String? + model_group String? + custom_llm_provider String? + mcp_namespaced_tool_name String? + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@index([date]) + @@index([end_user_id]) + @@index([api_key]) + @@index([model]) + @@index([mcp_namespaced_tool_name]) +} + // Track daily team spend metrics per model and key model LiteLLM_DailyTeamSpend { id String @id @default(uuid()) diff --git a/litellm/constants.py b/litellm/constants.py index fa9f1d527af..ededd35001f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -149,6 +149,7 @@ REDIS_UPDATE_BUFFER_KEY = "litellm_spend_update_buffer" REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_spend_update_buffer" REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_team_spend_update_buffer" REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_org_spend_update_buffer" +REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_end_user_spend_update_buffer" REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer" MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)) MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 10000)) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 53a8627bc8f..b8731bd2e55 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3615,6 +3615,8 @@ class DailyOrganizationSpendTransaction(BaseDailySpendTransaction): class DailyUserSpendTransaction(BaseDailySpendTransaction): user_id: str +class DailyEndUserSpendTransaction(BaseDailySpendTransaction): + end_user_id: str class DailyTagSpendTransaction(BaseDailySpendTransaction): request_id: Optional[str] diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 6c9289e3ff6..715a6ebd25d 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -25,6 +25,7 @@ from litellm.proxy._types import ( DailyTagSpendTransaction, DailyOrganizationSpendTransaction, DailyTeamSpendTransaction, + DailyEndUserSpendTransaction, DailyUserSpendTransaction, DBSpendUpdateTransactions, Litellm_EntityType, @@ -65,6 +66,7 @@ class DBSpendUpdateWriter: self.spend_update_queue = SpendUpdateQueue() self.daily_spend_update_queue = DailySpendUpdateQueue() self.daily_team_spend_update_queue = DailySpendUpdateQueue() + self.daily_end_user_spend_update_queue = DailySpendUpdateQueue() self.daily_org_spend_update_queue = DailySpendUpdateQueue() self.daily_tag_spend_update_queue = DailySpendUpdateQueue() @@ -182,6 +184,13 @@ class DBSpendUpdateWriter: ) ) + asyncio.create_task( + self.add_spend_log_transaction_to_daily_end_user_transaction( + payload=payload, + prisma_client=prisma_client, + ) + ) + asyncio.create_task( self.add_spend_log_transaction_to_daily_team_transaction( payload=payload, @@ -475,6 +484,7 @@ class DBSpendUpdateWriter: daily_spend_update_queue=self.daily_spend_update_queue, daily_team_spend_update_queue=self.daily_team_spend_update_queue, daily_org_spend_update_queue=self.daily_org_spend_update_queue, + daily_end_user_spend_update_queue=self.daily_end_user_spend_update_queue, daily_tag_spend_update_queue=self.daily_tag_spend_update_queue, ) @@ -538,6 +548,16 @@ class DBSpendUpdateWriter: proxy_logging_obj=proxy_logging_obj, daily_spend_transactions=daily_tag_spend_update_transactions, ) + daily_end_user_spend_update_transactions = ( + await self.redis_update_buffer.get_all_daily_end_user_spend_update_transactions_from_redis_buffer() + ) + if daily_end_user_spend_update_transactions is not None: + await DBSpendUpdateWriter.update_daily_end_user_spend( + n_retry_times=n_retry_times, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + daily_spend_transactions=daily_end_user_spend_update_transactions, + ) except Exception as e: verbose_proxy_logger.error(f"Error committing spend updates: {e}") finally: @@ -627,6 +647,20 @@ class DBSpendUpdateWriter: daily_spend_transactions=daily_tag_spend_update_transactions, ) + ################## Daily End-User Spend Update Transactions ################## + # Aggregate all in memory daily end-user spend transactions and commit to db + daily_end_user_spend_update_transactions = cast( + Dict[str, DailyEndUserSpendTransaction], + await self.daily_end_user_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions(), + ) + + await DBSpendUpdateWriter.update_daily_end_user_spend( + n_retry_times=n_retry_times, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + daily_spend_transactions=daily_end_user_spend_update_transactions, + ) + async def _commit_spend_updates_to_db( # noqa: PLR0915 self, prisma_client: PrismaClient, @@ -990,6 +1024,20 @@ class DBSpendUpdateWriter: ) -> None: ... + @overload + @staticmethod + async def _update_daily_spend( + n_retry_times: int, + prisma_client: PrismaClient, + proxy_logging_obj: ProxyLogging, + daily_spend_transactions: Dict[str, DailyEndUserSpendTransaction], + entity_type: Literal["end_user"], + entity_id_field: str, + table_name: str, + unique_constraint_name: str, + ) -> None: + ... + @overload @staticmethod async def _update_daily_spend( @@ -1015,14 +1063,15 @@ class DBSpendUpdateWriter: Dict[str, DailyTeamSpendTransaction], Dict[str, DailyTagSpendTransaction], Dict[str, DailyOrganizationSpendTransaction], + Dict[str, DailyEndUserSpendTransaction], ], - entity_type: Literal["user", "team", "org", "tag"], + entity_type: Literal["user", "team", "org", "tag", "end_user"], entity_id_field: str, table_name: str, unique_constraint_name: str, ) -> None: """ - Generic function to update daily spend for any entity type (user, team, org, tag) + Generic function to update daily spend for any entity type (user, team, org, tag, end_user) """ from litellm.proxy.utils import _raise_failed_update_spend_exception @@ -1267,6 +1316,27 @@ class DBSpendUpdateWriter: unique_constraint_name="organization_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", ) + @staticmethod + async def update_daily_end_user_spend( + n_retry_times: int, + prisma_client: PrismaClient, + proxy_logging_obj: ProxyLogging, + daily_spend_transactions: Dict[str, DailyEndUserSpendTransaction], + ): + """ + Batch job to update LiteLLM_DailyEndUserSpend table using in-memory daily_spend_transactions + """ + await DBSpendUpdateWriter._update_daily_spend( + n_retry_times=n_retry_times, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + daily_spend_transactions=daily_spend_transactions, + entity_type="end_user", + entity_id_field="end_user_id", + table_name="litellm_dailyenduserspend", + unique_constraint_name="end_user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", + ) + @staticmethod async def update_daily_tag_spend( n_retry_times: int, @@ -1292,7 +1362,7 @@ class DBSpendUpdateWriter: self, payload: Union[dict, SpendLogsPayload], prisma_client: PrismaClient, - type: Literal["user", "team", "org", "request_tags"] = "user", + type: Literal["user", "team", "org", "request_tags", "end_user"] = "user", ) -> Optional[BaseDailySpendTransaction]: common_expected_keys = ["startTime", "api_key"] if type == "user": @@ -1303,6 +1373,8 @@ class DBSpendUpdateWriter: expected_keys = ["organization_id", *common_expected_keys] elif type == "request_tags": expected_keys = ["request_tags", *common_expected_keys] + elif type == "end_user": + expected_keys = ["end_user_id", *common_expected_keys] else: raise ValueError(f"Invalid type: {type}") if not all(key in payload for key in expected_keys): @@ -1474,6 +1546,48 @@ class DBSpendUpdateWriter: update={daily_transaction_key: daily_transaction} ) + async def add_spend_log_transaction_to_daily_end_user_transaction( + self, + payload: SpendLogsPayload, + prisma_client: Optional[PrismaClient] = None, + ) -> None: + if prisma_client is None: + verbose_proxy_logger.debug( + "prisma_client is None. Skipping writing spend logs to db." + ) + return + + end_user_id = payload.get("end_user") + if end_user_id is None or end_user_id == "": + verbose_proxy_logger.debug( + "end_user is None or empty for request. Skipping incrementing end user spend." + ) + return + + payload_with_end_user_id = cast( + SpendLogsPayload, + { + **payload, + "end_user_id": end_user_id, + }, + ) + + base_daily_transaction = ( + await self._common_add_spend_log_transaction_to_daily_transaction( + payload_with_end_user_id, prisma_client, "end_user" + ) + ) + if base_daily_transaction is None: + return + + daily_transaction_key = f"{end_user_id}_{base_daily_transaction['date']}_{payload_with_end_user_id['api_key']}_{payload_with_end_user_id['model']}_{payload_with_end_user_id['custom_llm_provider']}" + daily_transaction = DailyEndUserSpendTransaction( + end_user_id=end_user_id, **base_daily_transaction + ) + await self.daily_end_user_spend_update_queue.add_update( + update={daily_transaction_key: daily_transaction} + ) + async def add_spend_log_transaction_to_daily_tag_transaction( self, payload: SpendLogsPayload, diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 921fd9701bd..e3b20d7266d 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -16,6 +16,7 @@ from litellm.constants import ( REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY, REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY, REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY, + REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY, REDIS_UPDATE_BUFFER_KEY, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -24,6 +25,7 @@ from litellm.proxy._types import ( DailyTeamSpendTransaction, DailyUserSpendTransaction, DailyOrganizationSpendTransaction, + DailyEndUserSpendTransaction, DBSpendUpdateTransactions, ) from litellm.proxy.db.db_transaction_queue.base_update_queue import service_logger_obj @@ -107,6 +109,7 @@ class RedisUpdateBuffer: daily_spend_update_queue: DailySpendUpdateQueue, daily_team_spend_update_queue: DailySpendUpdateQueue, daily_org_spend_update_queue: DailySpendUpdateQueue, + daily_end_user_spend_update_queue: DailySpendUpdateQueue, daily_tag_spend_update_queue: DailySpendUpdateQueue, ): """ @@ -172,6 +175,9 @@ class RedisUpdateBuffer: daily_org_spend_update_transactions = ( await daily_org_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() ) + daily_end_user_spend_update_transactions = ( + await daily_end_user_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) daily_tag_spend_update_transactions = ( await daily_tag_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() ) @@ -207,6 +213,12 @@ class RedisUpdateBuffer: service_type=ServiceTypes.REDIS_DAILY_SPEND_UPDATE_QUEUE, ) + await self._store_transactions_in_redis( + transactions=daily_end_user_spend_update_transactions, + redis_key=REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY, + service_type=ServiceTypes.REDIS_DAILY_END_USER_SPEND_UPDATE_QUEUE, + ) + await self._store_transactions_in_redis( transactions=daily_tag_spend_update_transactions, redis_key=REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY, @@ -365,6 +377,30 @@ class RedisUpdateBuffer: ), ) + async def get_all_daily_end_user_spend_update_transactions_from_redis_buffer( + self, + ) -> Optional[Dict[str, DailyEndUserSpendTransaction]]: + """ + Gets all the daily end-user spend update transactions from Redis + """ + if self.redis_cache is None: + return None + list_of_transactions = await self.redis_cache.async_lpop( + key=REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY, + count=MAX_REDIS_BUFFER_DEQUEUE_COUNT, + ) + if list_of_transactions is None: + return None + list_of_daily_spend_update_transactions = [ + json.loads(transaction) for transaction in list_of_transactions + ] + return cast( + Dict[str, DailyEndUserSpendTransaction], + DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions( + list_of_daily_spend_update_transactions + ), + ) + async def get_all_daily_tag_spend_update_transactions_from_redis_buffer( self, ) -> Optional[Dict[str, DailyTagSpendTransaction]]: diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index 3afbbdd5a4b..9ff0fe6e590 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -20,6 +20,10 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.utils import handle_exception_on_proxy +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + SpendAnalyticsPaginatedResponse, +) +from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity router = APIRouter() @@ -673,4 +677,78 @@ async def list_end_user( str(e) ) ) - raise handle_exception_on_proxy(e) \ No newline at end of file + raise handle_exception_on_proxy(e) + +@router.get( + "/customer/daily/activity", + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], + response_model=SpendAnalyticsPaginatedResponse, +) +@router.get( + "/end_user/daily/activity", + tags=["Customer Management"], + include_in_schema=False, + dependencies=[Depends(user_api_key_auth)], +) +async def get_customer_daily_activity( + end_user_ids: Optional[str] = None, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + model: Optional[str] = None, + api_key: Optional[str] = None, + page: int = 1, + page_size: int = 10, + exclude_end_user_ids: Optional[str] = None, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + + """ + Get daily activity for specific organizations or all accessible organizations. + """ + from litellm.proxy.proxy_server import ( + prisma_client, + ) + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + # Parse comma-separated ids + end_user_ids_list = end_user_ids.split(",") if end_user_ids else None + exclude_end_user_ids_list: Optional[List[str]] = None + if exclude_end_user_ids: + exclude_end_user_ids_list = ( + exclude_end_user_ids.split(",") if exclude_end_user_ids else None + ) + + + # Fetch organization aliases for metadata + where_condition = {} + if end_user_ids_list: + where_condition["user_id"] = {"in": list(end_user_ids_list)} + end_user_aliases = await prisma_client.db.litellm_endusertable.find_many( + where=where_condition + ) + end_user_alias_metadata = { + e.user_id: {"alias": e.alias} + for e in end_user_aliases + } + + # Query daily activity for organizations + return await get_daily_activity( + prisma_client=prisma_client, + table_name="litellm_dailyenduserspend", + entity_id_field="end_user_id", + entity_id=end_user_ids_list, + entity_metadata_field=end_user_alias_metadata, + exclude_entity_ids=exclude_end_user_ids_list, + start_date=start_date, + end_date=end_date, + model=model, + api_key=api_key, + page=page, + page_size=page_size, + ) \ No newline at end of file diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 2883dfc4b82..4d4a127f7e8 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -462,6 +462,34 @@ model LiteLLM_DailyOrganizationSpend { @@index([mcp_namespaced_tool_name]) } +// Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyEndUserSpend { + id String @id @default(uuid()) + end_user_id String? + date String + api_key String + model String? + model_group String? + custom_llm_provider String? + mcp_namespaced_tool_name String? + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@index([date]) + @@index([end_user_id]) + @@index([api_key]) + @@index([model]) + @@index([mcp_namespaced_tool_name]) +} + // Track daily team spend metrics per model and key model LiteLLM_DailyTeamSpend { id String @id @default(uuid()) diff --git a/schema.prisma b/schema.prisma index 2883dfc4b82..4d4a127f7e8 100644 --- a/schema.prisma +++ b/schema.prisma @@ -462,6 +462,34 @@ model LiteLLM_DailyOrganizationSpend { @@index([mcp_namespaced_tool_name]) } +// Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyEndUserSpend { + id String @id @default(uuid()) + end_user_id String? + date String + api_key String + model String? + model_group String? + custom_llm_provider String? + mcp_namespaced_tool_name String? + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) + @@index([date]) + @@index([end_user_id]) + @@index([api_key]) + @@index([model]) + @@index([mcp_namespaced_tool_name]) +} + // Track daily team spend metrics per model and key model LiteLLM_DailyTeamSpend { id String @id @default(uuid()) From 2e65c464ade1a0dd5e86a2902f2e2fe1b01f1168 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 4 Dec 2025 12:36:15 -0800 Subject: [PATCH 02/40] Adding tests --- .../proxy/db/test_db_spend_update_writer.py | 75 +++++++++++++- .../test_customer_endpoints.py | 98 ++++++++++++++++++- 2 files changed, 171 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 181d21b44f6..db6c318357c 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -572,4 +572,77 @@ async def test_add_spend_log_transaction_to_daily_org_transaction_skips_when_org org_id=None, ) - writer.daily_org_spend_update_queue.add_update.assert_not_called() \ No newline at end of file + writer.daily_org_spend_update_queue.add_update.assert_not_called() + + +@pytest.mark.asyncio +async def test_add_spend_log_transaction_to_daily_end_user_transaction_injects_end_user_id_and_queues_update(): + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value="success") + + end_user_id = "end-user-xyz" + payload = { + "request_id": "req-1", + "user": "test-user", + "end_user": end_user_id, + "startTime": "2024-01-01T12:00:00", + "api_key": "test-key", + "model": "gpt-4", + "custom_llm_provider": "openai", + "model_group": "gpt-4-group", + "prompt_tokens": 10, + "completion_tokens": 5, + "spend": 0.2, + "metadata": '{"usage_object": {}}', + } + + writer.daily_end_user_spend_update_queue.add_update = AsyncMock() + + await writer.add_spend_log_transaction_to_daily_end_user_transaction( + payload=payload, + prisma_client=mock_prisma, + ) + + writer.daily_end_user_spend_update_queue.add_update.assert_called_once() + + call_args = writer.daily_end_user_spend_update_queue.add_update.call_args[1] + update_dict = call_args["update"] + assert len(update_dict) == 1 + for key, transaction in update_dict.items(): + assert key == f"{end_user_id}_2024-01-01_test-key_gpt-4_openai" + assert transaction["end_user_id"] == end_user_id + assert transaction["date"] == "2024-01-01" + assert transaction["api_key"] == "test-key" + assert transaction["model"] == "gpt-4" + assert transaction["custom_llm_provider"] == "openai" + + +@pytest.mark.asyncio +async def test_add_spend_log_transaction_to_daily_end_user_transaction_skips_when_end_user_id_missing(): + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value="success") + + payload = { + "request_id": "req-2", + "user": "test-user", + "startTime": "2024-01-01T12:00:00", + "api_key": "test-key", + "model": "gpt-4", + "custom_llm_provider": "openai", + "model_group": "gpt-4-group", + "prompt_tokens": 10, + "completion_tokens": 5, + "spend": 0.2, + "metadata": '{"usage_object": {}}', + } + + writer.daily_end_user_spend_update_queue.add_update = AsyncMock() + + await writer.add_spend_log_transaction_to_daily_end_user_transaction( + payload=payload, + prisma_client=mock_prisma, + ) + + writer.daily_end_user_spend_update_queue.add_update.assert_not_called() \ No newline at end of file diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py index 86a6ceec25e..25ff6f89427 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py @@ -1,4 +1,4 @@ -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import FastAPI, HTTPException, Request, status @@ -301,3 +301,99 @@ def test_customer_endpoints_error_schema_consistency(mock_prisma_client, mock_us for key in ["message", "type", "code"]: assert isinstance(error1[key], str), f"error1[{key}] should be a string" assert isinstance(error2[key], str), f"error2[{key}] should be a string" + + +@pytest.mark.asyncio +async def test_get_customer_daily_activity_admin_param_passing(monkeypatch): + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints import customer_endpoints + from litellm.proxy.management_endpoints.customer_endpoints import ( + get_customer_daily_activity, + ) + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock( + return_value=[] + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + mocked_response = MagicMock(name="SpendAnalyticsPaginatedResponse") + get_daily_activity_mock = AsyncMock(return_value=mocked_response) + monkeypatch.setattr( + customer_endpoints, "get_daily_activity", get_daily_activity_mock + ) + + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin1") + result = await get_customer_daily_activity( + end_user_ids="end-user-1,end-user-2", + start_date="2024-01-01", + end_date="2024-01-31", + model="gpt-4", + api_key="test-key", + page=2, + page_size=5, + exclude_end_user_ids="end-user-3", + user_api_key_dict=auth, + ) + + get_daily_activity_mock.assert_awaited_once() + kwargs = get_daily_activity_mock.call_args.kwargs + assert kwargs["table_name"] == "litellm_dailyenduserspend" + assert kwargs["entity_id_field"] == "end_user_id" + assert kwargs["entity_id"] == ["end-user-1", "end-user-2"] + assert kwargs["exclude_entity_ids"] == ["end-user-3"] + assert kwargs["start_date"] == "2024-01-01" + assert kwargs["end_date"] == "2024-01-31" + assert kwargs["model"] == "gpt-4" + assert kwargs["api_key"] == "test-key" + assert kwargs["page"] == 2 + assert kwargs["page_size"] == 5 + + assert result is mocked_response + + +@pytest.mark.asyncio +async def test_get_customer_daily_activity_with_end_user_aliases(monkeypatch): + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints import customer_endpoints + from litellm.proxy.management_endpoints.customer_endpoints import ( + get_customer_daily_activity, + ) + + mock_prisma_client = AsyncMock() + mock_end_user1 = MagicMock() + mock_end_user1.user_id = "end-user-1" + mock_end_user1.alias = "Customer One" + mock_end_user2 = MagicMock() + mock_end_user2.user_id = "end-user-2" + mock_end_user2.alias = "Customer Two" + + mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock( + return_value=[mock_end_user1, mock_end_user2] + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + mocked_response = MagicMock(name="SpendAnalyticsPaginatedResponse") + get_daily_activity_mock = AsyncMock(return_value=mocked_response) + monkeypatch.setattr( + customer_endpoints, "get_daily_activity", get_daily_activity_mock + ) + + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin1") + await get_customer_daily_activity( + end_user_ids="end-user-1,end-user-2", + start_date="2024-01-01", + end_date="2024-01-31", + model=None, + api_key=None, + page=1, + page_size=10, + exclude_end_user_ids=None, + user_api_key_dict=auth, + ) + + kwargs = get_daily_activity_mock.call_args.kwargs + assert kwargs["entity_metadata_field"] == { + "end-user-1": {"alias": "Customer One"}, + "end-user-2": {"alias": "Customer Two"}, + } From 5439f03bfcb74175633cf57a3d9b75af3854fbd6 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 4 Dec 2025 12:56:43 -0800 Subject: [PATCH 03/40] =?UTF-8?q?bump:=20version=200.4.9=20=E2=86=92=200.4?= =?UTF-8?q?.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 6bc576e4f3d..cc8a92b9c67 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.9" +version = "0.4.10" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.9" +version = "0.4.10" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index 81e31a5ea81..da31bc9d8ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ websockets = {version = "^15.0.1", optional = true} boto3 = {version = "1.36.0", optional = true} redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = "^1.21.2", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.9", optional = true} +litellm-proxy-extras = {version = "0.4.10", optional = true} rich = {version = "13.7.1", optional = true} litellm-enterprise = {version = "0.1.22", optional = true} diskcache = {version = "^5.6.1", optional = true} diff --git a/requirements.txt b/requirements.txt index b61428588d7..ac1eba2f4c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests cryptography==44.0.1 tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.9 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.10 # for proxy extras - e.g. prisma migrations ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env tiktoken==0.8.0 # for calculating usage From 183437795042392776c3ef6e8c798cfeafebb9c7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 4 Dec 2025 12:57:06 -0800 Subject: [PATCH 04/40] Adding migration --- .../migration.sql | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql new file mode 100644 index 00000000000..c4234785c54 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql @@ -0,0 +1,42 @@ +-- CreateTable +CREATE TABLE "LiteLLM_DailyEndUserSpend" ( + "id" TEXT NOT NULL, + "end_user_id" TEXT, + "date" TEXT NOT NULL, + "api_key" TEXT NOT NULL, + "model" TEXT, + "model_group" TEXT, + "custom_llm_provider" TEXT, + "mcp_namespaced_tool_name" TEXT, + "prompt_tokens" BIGINT NOT NULL DEFAULT 0, + "completion_tokens" BIGINT NOT NULL DEFAULT 0, + "cache_read_input_tokens" BIGINT NOT NULL DEFAULT 0, + "cache_creation_input_tokens" BIGINT NOT NULL DEFAULT 0, + "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "api_requests" BIGINT NOT NULL DEFAULT 0, + "successful_requests" BIGINT NOT NULL DEFAULT 0, + "failed_requests" BIGINT NOT NULL DEFAULT 0, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "LiteLLM_DailyEndUserSpend_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_date_idx" ON "LiteLLM_DailyEndUserSpend"("date"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_end_user_id_idx" ON "LiteLLM_DailyEndUserSpend"("end_user_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_api_key_idx" ON "LiteLLM_DailyEndUserSpend"("api_key"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_model_idx" ON "LiteLLM_DailyEndUserSpend"("model"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyEndUserSpend_mcp_namespaced_tool_name_idx" ON "LiteLLM_DailyEndUserSpend"("mcp_namespaced_tool_name"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyEndUserSpend_end_user_id_date_api_key_model_cu_key" ON "LiteLLM_DailyEndUserSpend"("end_user_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name"); + From b12ccb1a7acb21567f46e3b149651bf13fb6502a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 4 Dec 2025 13:11:20 -0800 Subject: [PATCH 05/40] Publish proxy extras --- ...litellm_proxy_extras-0.4.10-py3-none-any.whl | Bin 0 -> 40415 bytes .../dist/litellm_proxy_extras-0.4.10.tar.gz | Bin 0 -> 18851 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10-py3-none-any.whl create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10.tar.gz diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..ce4e805663a67d925d8007a8a098125eb98a0cf7 GIT binary patch literal 40415 zcmbrm1yq*p(l$(YcQ;6L(;y&{(jC&>%?;Ah(n@zoOG~44cOzX=(nzQBUFfr)_tX9C z|8xH@YYi^flI1(oU^W8-D|;&!0|Qn^4`>LbUw(d}JU3(<__;91|F567b+E9ovbO+!UR_bP zyaR-?_ytD;Q>E%Hy!*sLPDqepJ7lzkyuSDobz7wmLbP^Q?s4ws`^ODqh0LpQwXE9t zc#ahQHt0f`td>Qf_)lkT4(fG@raGV{_eUE(d>8zvm(-}SF~qrxgvPNRBDZfUPvfuQ z4Iyv0kNouDIguoY5iK2=OASk9npx0e5j$6cY*f|0#6yRB?@Z%PN}QCM7sV*9K@q;f zWp?eFnVF2bdqy7WA!pyfu)cMgnkc=zXRsA#uJCxPn0G|g;H<&u)@AB^aoE0@cNg+k zI6`INN@w99Af9nSKuG*2IPI(~oQ+(p9PGht93T#E5GMx@I}ZmtJEwuM5!lSu%HE9a z?|;GycCy{nUUFFA!SEe5pxnqui$;OHm=!36BDjip-;_VukcgQv;|61`0{UyxM7h|3-_pxZ9LSIqP-kxTiK_W;_2w@;*h&1KRZ#GA)18{tEk>C{6-MlMYsBWc z%img7nPEMBO3*Pz{k4Kc&dSqT&Z7K$+*yVyRa`%msw?qB`b$%jO}CsYvd?~H)61Wa zBChvO(y^Imv7TbSMn1k`hf=-3e;;;uHm}{Vbw0d3Qytqg`iT&sv-VYFn5!A9-3$R6 zX>aNWefdvXM&~p$2Rw+G{n^?g9h8pp- z+*UBfMpUQ{wTzClIE6Sx!sPiTmK5Y9;xp1nk?n7WG~%lBc`cdKLt$Mp)i`i-t!0s*{Ji6mHo>9V-pHUL79Esd@-h`h)R4 z%96TIW|5w232K6w7GN;L8~yRyEn&mfN-q7jGoCrp)0tVBqKiOA?Sd8MFoHb7kj_7F zZ!>m%7cyPb8~IXaCX=UKosN!6#%_3VR_ z5F69+BO#DH%qJ{K5U?s(o#*VoLPyY$U%8E)I?1IZx|W?l2S|nCdWm(t4jcGdnk!9| zsm_IhFe*$bVzYtoiq;b16!$3N-c)4Zc|96q*&E~4I#1G}cYG*c#Y)3Qyu?ww5o(-g6ftQFQo8EtxU)oW3^lnOCbHKdH?ZusbGvKPO=>N$fD; z*-AT7xRZe~YHYfKt=pCM8J$6DI<4lo~y9SIkna z&(M(9aXm`V`@Wp4jIA;7!56|bS6!y#895znUFaAChh?9tj8fB1h$_-Hk6Ezxd$5x$ zRVm4;pF)D!G+}`&Z0?b+fDPtO#|=fY}C+Vx^5WU(?9M8%A;BPEtfeF-xrc^104a_dK- zTiA9uD_~8AKrzC6vag(ZllByv8^vIoxi?DnA+E&Z;hih1$i&^*itv8!^5m=5J&SOm z9MMO-^VMnl-$_GxIXq}w$<$jJOEUU{TuZBBh*X}W>)aoyEw=caoPCR>m15IUSS zYLQt;eU;V&>0OFapDus@TzmhHz4FPCyZE?AlFIgL%H*5zK`Hk^4j1fU9|?8VFxxbN zwhI$K|H#xpkG*A)jZZEdl8dmxsd^!bs){q*ENSYWb8>~nZ&ZLwgvE;!1(s@i2coCc5Q{Ms2x2_&dkq_y2N zyo!ur-94v+&sIhgzrM2@tUps11s@#dzPG%5+k|YbeSJu8s!HnRs**!fKS)y+L0Oc3 zFj15(X!%4iI#QU8$KVax>yE<4CCG%xjOv4ui_b!4*4pqYmD(A>CUK=!=6$HsXOH!t zR^9)`B)F?rnM?o{kp=?+q5R*JPF{8ph{M3h)YQP#$ja8kz!hxfY~W&KZ2Mao%}~{` z2?P=NTx!B(G0(Eku4ks3Q!`uRQ`acAvlvM=!_?+}yNu}J>#2!Dg1ol>(wJVo1l5YE z^~mxl;W1ijo*!ne`~uNHwBsdZGGy6Ykzo-X;o3HGi$2BOEYG(-j6r* z#nW)~IbUI8iykAaRwo%F9m)&g>W8vcl$`UI=|9IP{3hxKb3-rU#1-er+m7;Om3dFC zo<0ciyd&fE{dL-UVNhFW6~n;3!O0%3t_{N3I(>GRsu*)dYX4^nfdJ7rByCKB5QW0G zJ@bOv)6bmKuZb~?9-;CQ^IJ={&QK#?{5wB#qK4<3dm1phH*!E3L~ObWV?$j5v;jo24+Z7wADUpY{|si zAQ@rmrCl8@5OPpo38`CVE*H7GlkH$ch#?#?pszAakGzKZ)lXe@j__5$zjz>e7605% zeh?oQABfw)-ptLw#M!~#z}mt1pW@avTEr#@gb{pnjVs2=xLwqnIpKn58&}#@ITdF) zY^2jTK!pD7a=7l0idg_7*=YIJJ%t(C!@11OUAHBrh91Fx7s|giQg276b%;b9(UK9> ziH2s=fgm4a)BGUrSx*)6t9g4z&DZWKka=j!4Ee+!L~kGmD9tg#eS@yS`>A51{$DfN z_B%`BW$rI0XIGYnZK~`BB49F|9x+O!MV8oDTC6auk_Tjd_|Ov@#JU!;G0n_mGp+ zBA=ZYS}}-DUD!XL5GdZbRh{mKdJJFq+L(AXB7>JrHEUB{B0{?X?jq>``|&G`u(oSX zN#oh0A%{{|5((Gxor?DMWb|J_NYx_2Xfosqqf1>m&ni0e90Nj;exBKw8LUAh9XLQ(+8*6-1AEZE9aKsxtLGR>gZ;sO7g8dm|v}N_Z zvsS!3lW3w_ycRrbZU^Gjz^_Q6S!m!H#rE=t7K~)jDjI zEw)g%8(N}ww@g@cqBb~qO99o2n70f>EtI@f{B$^|m+L=jG-!a4|EoGq5wWGX^S-qnWdv6&U=Zt@}GZAKEwCevyZ)B>CA# z;W*BRO-a)PTT?X{1e+f}*1-%FB@=s_`>EdBV8;KVKw{cXQ;hk^)9oR$L{-%-_VDuS zULYa8B34|-eOBkgU_P9dcr_iXTDEguinn&e!RGi~6dRcy#&bw^<^I0^_b}ok$Y^*8 zc+~-z)1Q0C&Cbov!^Qg}j7*G7EX{z+V`OUJ<_u^9KoYhwvj_i`h{29#zzqZLoITjc z^&sGBgM5f59)f04Vs``8jxg zfblbR{5bS~;umTU`~v6YPky2Nz%Pa+lap8_EwchRccMMhdd9!sO5`r})Kud_Ioq%F z_g`jgV$I^dfq68pGr&zO7V-3H=`(n!C21HplBDun8X-(FGm(C90kjzO$t3o*>}n#! zXwoAW64klj$UIdAy2>6Q<&-gK;V6OXfa9}(h!#x=)ndP8GTvHxj?@A|mXbA2qo$k9 zq@Ge7H5~ZNna5Ab{6{geN&>mPt3*TmJ}D`_D%7Qscp_Y=0j+auBrt~hiU%qBD2isW z>}6#38aZl3H**l(YY)Anw6oOGW^d(@^8QePL=X`ozxYOn-}A$s27|ld_hOOR&$juC zE5_pYonck?pzKzV5$a5zsiNeW^~Rw}mIu47!XEmKS2<;~ka5}ZD-mr*lEceTJD}Fb z!7hx@n7w%%{f4vKODyJ6BT^ra(-01FDSV2_3U1=<2c4Msi|@V4CmE86LT5xHq=uKz zJ4w~yEy_wzIcz?fw?+;feKJ9d=piv6KY(rd?ZDT`Y-GEE18)HI7^OdtejavqZqC2x zhl|l)RpJNtF#c7R$ybwi$OU0EpE8xEV}+G69ylqox@%gZnuf*9$8D}z;U!eSR$Jfu zM9I%f(jrRpQ;rUKz06Fh*QeH{w@h)TZ)=Z_fnoQSMbsYCh0LeCl+bSiVQkj<%dlcx z_-u4@p;B{sN64A_`dhFdB+THBmK12}eo|C&MaB`2yqic?9#V=|3HLt1E{xA6o=Jh5 z;B)5bL+{n9h~ae%OjxCb;p8Y2*_T9=?Zw!-Ns1A;fZLx3)4rI5EV3$~&gs9#Zjyr^ z)E0H3>T)dE=$jdyUa1t4WnrZ;{5JqFHAV5@I znxngRdVJ;>hru=r=ftJ_^wEk;Fh{CY(b~Vb!!%8;RBF`x?n3GtfQ>c&P^h-?? zG&-+3hfi*I!NEy%h+zc2gG87&l9`f@q;cEb@PZFXpankgqaY9}j{(#DZ$gHfos*N_z|`5n5va8m2Cnv20B5y-pdb(I)ymn-&dlEB*YKJA zr0&Qt!1i@WR=RA}8Sw;N>FXKTDN0G{q@P*({P;M^O?qgk2t`^BomFaBHjGVL?#-w| znOfD>8q{xrzBdhMhQOzF1I+rr`P6*e>|8%Hn~|-pgWCg3a&UDq0~`E{biFWhwlcRe z`5{*R%>2LlK->4Dw84%6rLDgO|6R#=*C#bfISGb*aZ*YGqEO5Vwl$rWa_G$h^)>Di zN<4Ck8Z-34B8JoV=`{t%91u{i4XjLmbxUvX z2MdH46Ebiz-7#)$W8ij*wn4r?vKQH*^q-h&qz}J3TH9SP&KRfNcIuYSED0Ins>`t)+XNz ztE>lP<>FwG9?cV-!v5-^m+Ud*3EvdadB`q00ZC5 zz`r^h95Za)3Bm|E@(h=FM4YPnMu?#t$(~lA&)G^rv^J4ySJ;+ztiEC`&ws2&Yj)B* zbxEsCBU;`P+WK~xsvpN9HPGCq-5-fu3BfAw-~j%krfmmEz0yhF5ed=%xl1d(`4g!g z0i@JcEBb-iS`4>m@8%?$IqeowoQN)#Kf91NIqunoJljv*b+o4wh7&&X6(fb)vwe?~ zSyJ`Y&HCuNz~Ny`lC%39>_B;a4$B}-j&Ke4E6Np2bIK1W-vcQB zahc!-@v`%`Wf6Js>#zQ<=z6mVb~PT5g&2o79X%D{Yf9s+gg*YOttF3!`_D zd#52u=+|MJRRBMHtj{+UHU3tgT@wJpMWd zBM0IY{zptzJ>mdU2dwb(uQnq`3N?oU2MSY!R^!}toGlAS=2z_(_0&zD1S~G54aUC? z=}QgMGN*TTYwe@(SX=hVAM>8=a5O6>meBY#%$~KkGWl}x*%T2bO)!4bdpxK`E3GeC z;x3a@i%}YFO;r)Ouqy=ZJX)rTDL*E$yBTrl%M50; zwO)#=-@4;_TGW(e0AEr7Uz0zL;(y`$lZ0BD8QHp68URcb={ba<(u=7`t^Q+AC-8iHF!3~(6kF11AZ>_h zOl>Wd>(wsZT}+AJfJDt<1wTCEhXL2u7>OP;;RqnSmU8hAiuNoYpjFADgBk>z;w$%a zQjyCgd&6`)An^-V$WE!Ksc|C zZgE2N8McudN|=R6jpbqD`i zqLb2MU;8Xs95Pmr112j)g_B1kMz1jCT1qy~qIL)#3+5?xVkAEaxv;O#WQ58=?NUn4 zJC^}wpXe(pa>0rfebHT)%V*QlPB(YA)1&jL30ho+Cs8BzY$9a}vP)BMp~A#NULJt> ziPJrI+*!W2~Rk)8CNjk9L?iJ8S;cch{b{s#Su6%hT#Lvo4>>nUbW6N^2RA9=w3xP#;d@qu_b*g>2>%aDtai|dbw{?ShX!_wd4 z`GG17F-Wr@Ny`mLZ#thLI-#l{IJulSssBwDevpLYenmD76-O5*Wygz4#Pf4i2*~`k z*sfTWx9*`f;-TF5_`flS zmzOkO1VkXjwe|^w8P5>Rj>!lRtx%pM;4v)WU?%PMTro_}#y=6X*U(9IQ2FXQGd+)2 z*KBCw-DeDkX#_KCgfEs{8W+#fWFd|cB+9hVgSx&JDG}1dY?|xUJA{b`EwuGfcvTy1 zZiXx>Ln>JxfvaFQf%-YBkI6NZwx>Qwo%G~p@bG0>bNy(iRXWR_>(lEIlH1b zssBbB|Bi&$Yzi!_BSWJRqjHcrNVKrN!pz0-T=5vY@myse8h(p}oab1Idw>f!KtlRs z^&Oy(fEye@Bs{3^7OqClrp|!c288mztM5KBA`fNo6`*30^D|6Mz81*mA>J|GUwc$j z7|pU1E%9i&ErFD-I* zYIVLSoZ!&5_uIYR_KeRUugOFyW8sKpMglw70xIJ*Ie7-Q*yLX6ZUTkGD zP6d_WXv*fW7l`QK^Aw1)JNXLR7AnAO;ia8zZY_dGg}!`beOszToH@j4M%lp*{rUzb zP<}!b6Du5~MOz_*HX&yfkJ@+H9w9}d!YAlesVtqHm+U*=cAV^1#%P>E8+`R#HX+Qc zpIYSu_t*}Kx8PIu&wd1AaXY0moaZ*NdaP_cg4&6D0fiyN-^e;c*I&WrT&HQ{tT6E< z7RcpiLlGqi)8*OhHmZM=8&tMgqE0n5&oy}JX%txb*$QX;B@5Mx-O0JeCol=Af(#N_ zkb9Rhp9|UJM~nRuZ00DctP1atI{9;v6^X-`8o-U^UYMj#ub>7N_VKP88E;=PBVeYr zac25~-?x~!zBDH&&6_9ZjI{cSWrFbTO>fzvrZS=sb50rj>Xf)QJbOZ5gR;zdWNPqT z?`yTgBir5;p9Xoc4D{b*Y5Z|7G6&#(ApTeP16b?;E;$HLWFPt~1AA9nTLYjGv~uzI zx1Qqu(sB)YliW6C{qCV*%?iqJ{&Cd1@Mui$6wwOzqhlM@Pxy=10gyw(!VV zsyEhJNlh%6;NAQz&ba_1LICBEYT%Y7$t%}0_(B#iw_&f9=gAmGnTbzvIkN+1^wUP0`$}z+L`Q>d)F(%~`TE zZW_>p6c3lJLHL}kx4ccNK{Y%^UKKU|=7dzO0GrMKydZe4uQHnd#RT2PO>~TvEww*B zRAt*@=)^H*d=k59l@dS7=It))bDR9Mn=b*myZYsW1!Fu^a;v;Uiu7Ea753t=p1WV= zk7tnH6<5XbPu1Q8l(aCut2a{)oU%gW;3XXi*ansI_mUw>w>>sQ(aull_YHzfd(r+v z31jfn_wX8YrpYvlCuqTYqZwbu?E>|?_{~K#m4w6^JMTeU3+~BAwzW^$46|3u2|piw zk6u7c#vkE7)@u>Zat})ypN%JrFSIUVeZb}=35tUG(Sgg2yO1lN_y3> z|p<^f{^j1QUN&7SC@j_{n#iTP;@gbZ;SC8ozwmMWxjF~OccmW zW;-ZaGGbt%y#Ib*cz-wu9$R;0eiftYHv5$}Ze!d(iAfYAo{o+&@Qb;@Nz3N(S?p`5 znra=TX|?74+T&LLY@y!Qh0(^d&BqdgrdpYS`;W+cCNJ(zE35dqNPDTyz4Kw1ldIH{ z;JWNtJH}d0PF-%ho}(AgeLU#B(j+fB7j}939&P3g9M2Vu+gdg5 zA|~ACiN>V1F&TB`p5%CzY?3f&F%&ty@PhvpTnUO*H5CBV2O;KhpD?Bk#+V7kuwiBWjqjb?Ig35eRb#;Zn6~mh zG%Y~9{Ormfnolcxa|Z(x2U}M=AWHsIJN_rWxS}B_urLM|EC1%j;zq#LP*aUJEo_CM zuka&Sd`;e7r7R5C^IdJ%fA@-|s0~&Ei2f(=5D;2`XMa@39=zfNfj~bs!3QtwjqJ?8 zjz%VcAnM}aVEg0mzlMBw{HQfBlkYitBGFeAe2(Z@wJXwRZ9~1!lJ_{2i(O=?>h>ba z_oFO&L0XjrTMEYAW*Qsk`Yru#yuv(^f_}UO8@+Vyl^@6oj>2@5YB`j1^E~ZUoiTKU zRyVw%F7H0{h+Hue4AYx$Z)vbaIhKl!A%s0LeTj)1bXVz&<;|MgqV&$jhWB&BNM`*B z-saxv`Qc*_eF=pdrO|hS&7}xeN{%tk$~o?=nkwDI@*I{0g)z_;2uX518C&Ym{oWRb z5?R1J5t&m%(IwGS?@zqfkldW{ddD+ocUiq75?uha$v&f&+zT0Xu=Ku%EBHkRz3j~H zrjg$8dT}h3ynciM%+hkEhAUj48^W7BCp|+GKH(_DmsG-ONFR3=h1hQ89BZ#8klik4 zedOjgS(5FdTF|7PaDFm>e0IRh8^~RM zjcdRzrdAFQaroX{!IMk##Q_Lmmy9u;}c5C~U2M*_N+5LuML#axhw z86K(rd!$wYB$hbivZ-?y_$!31jN|oo<9e+f=h`o@{Z)qoI-MuH1&6sSaW9)NM6EM4 zi^-275|+}<>Ee{RVHQJ!F+gY{0S&w;d%oTp%XtK%Mf2KwACGvnXU{6t`)_ZNe@#SW zp@g;i06u1b8U8s=e89!!W&hc5{(UWsoQ>>$2oJxow_njCjc7#j1ED}%zLeNL#OG5# zpehQ`&qa@ zk(|27+}bM9)mSWAjLAoav17}3w{;B8`TD>5c6cmWvY*01M@+lEFSqz?0e z8;<(}6F2b_lhanCPC5{qtGuL?JTGU%6;Ik5^MQCk z{JFpOZ`w;f8@a*Y5pHrT~xFpu%(34b@Vcv5}^(rxmm;0@q7+9 z2Ty)ynMSXA!Ou6`NmlqJ_lqPvq=QD76OPlg$Rp~>US$Q(Z6^ZBcj$I>OXTeRNg62v zk;O?F0k*YV$#4YREhUw8NjH9z3#Inv{@D1AdKO~nR)>a}T#yG7GHMOQp{g{F5qZoU z+b`i<@wC#umF#W|zqC};bNJenAFFXD(-6{op2s)3%j{Xotix=BZbWACNiTCgdYQW` zaT;YXNLL9F)1J@&n|+Ux3YmG(yH!OkH1j9|amb5&s)Z8F*6MG;~04qB+d}F9(~!06kAzm zIc5&8;rBf&n~z65s>wL%1m>fU)aAx+x|tUu4(>Yu9XcRF6@WkgC_;fn40e7lUSQqR z)&3zuO%48z;a`(hLUcQ@bA$5g$SZs-E&#ecw{S$AuqMrtZdsC)Fg^jVPT}i8StuC} z4ETG?#(i8to84Q;^KO+6+Hh!!(L4w`c?t$~a8A2R!KZfjkd{ZieqZ=)8MR8V$U4t( zyg9GlLbLZ3%OL5Cveq>vB4pH2v&1SE7A~lU*$|Tw7-n~2ewZKCICWUuWxwle9iLWY zosbAK#vg1}VezH!>3*YI8^zm2I3w-+8^_d69xZ)Xm;?i+{)d#{oXuJN4lj8xj1b(jnndSUooBvTU5gMpO@_X2(TB_h%{1!l0me{~GU|!VF{vPN2 zK8(oo6dTI)UHm~EyfK!Ya9{$G*XlMCWu7OV2@D32SCzH3$C&b`gPI(WG>J|F75br~LEwV1JEtd4 zw6f%^1oTqE_6i(Ew{yUb>3(W_{pO18%~<`x5anmJyCEGnf|yTgx9&TdVWdB(LG|h` zS(Gg(?%i=O7fW;`XSWr}0YGB!br`0H#Ia1frWM};M0}pv62d_(d zT#VMQw(m%>v(KK(Ocgiv)trjGRp@e>OP(aFKjsgd9342!h;Rb~DlGgcrWf z2ouFr<@?9?f13*_4b2BRijM<6dUW&&?kxNWwd)}X@e42H zrG`HZxQ~CMX&t@#Z4C(Ts(N7-K-CaH_fN$^V3^1LfGt2nABG`6_a;po?9GAs*gwEK z94#LKEQLOdL&R9?3>Cf3;?1dDp{P*cp*kfF@|nOj70wgQ*F9cRK^`&RzoUC zYlhTTXrlTW20d-WW95*SG#EXu=<4PuU}_yTs?0kpEw6(@_H>oyE^QRKELhYlpB3?o z$tjaD2{7WkNzdMw_#$^8hYn1Zn!wzQ4 zmkM`xmE=NFHyJxPt+?54Q((w6?b0_D4t@{3@kp_I#?al?=zaZCa53*Q5=-H+@U2X?1~?Xocu-F9)sLL`qmf{r>5^D6^+3)69sPF?Sk)6d`2j6Uat zO-YPJFdHdl3RxOm@x;FSlyDhU;g0XoR;`O8W6U_!#FTp2=@C+!M>5D#+K0DxoP#Kn zi6Ns%3)@~q!?ImM_q03K*1SOXBZ!g>pHuer{S06E*>62!)1%FKLjZ3J!2a|_0+78q z0VM>O=lv~KS(*H+;{KY%Sq3W6o`P^-$Cfk~UD&UoCUd#_m_&zFnu5C2q9pDbrP8lW z3AaPB(zDQ|Lo;?hzcYmH_QEk2n5!bD(Y;ruC{wNzoQETKl_bA3oA0W?QV906OoN(N zE2`Zb7L=g#IH8r}yUtpnH}46db zwVe#CAO4sh|9WJ9;Ms5ErC-+wa;7>arq*JwP zu|t}tF{>IglT%MNZI)2TEUiKpe#i3q?e@B8J4dD{gF6Cd$%&xyXfsuqoCGbe0R@w5 z4Fzx4w?jmh<66R5oM8b4tqi}9hlU8~S^m^zQUL?LNL32{)qbN8|R@soKDEy3#@jgA|j`+si`B@&ivmZ z-b1bQvYHvFH)303e)A2;E)pFC&rI>A*}rZGK;&*>%Hk_%6{&X*q>B`IgHcxzaC?>(5`SIZsdo5N|=LO&~>q{|p=|A!&F4wNr; zoV&Ku(AQiAF=Ci*4fSobZtX>#*>E+J(jQjyq}g9Mz^~ux8M^d4?4!R}&k}TCy+!=> zLbEZwI6VL&o)f4Ff2#8Q3>9G40Em1G;3@+>z`ySpZjxvJ0^znUC#$1Y7A^lsDz_frDw>;4*l#>XlWiU{RioM5ZgFFcj-8k3L5CEWBH+%OG9c{)io1UyxYRxD-hI zP3Z2OQ}WTpwETFW=B%ZLUDcPnjzI1f4r;D=Nrw1&{paXNw<~7}D}ncgojK8t$%tkw z#AO6YkFV_nyn`D)C^s2f!yh#S?%!OeZ?Fa7$ObfBezuSvllpFi(V)A|!{QUwH$qn{ z2cNn0%7f?bw|1J2`SOqg!1TfXR6PYS03L{67r z@YG^JtW5CY&{CE302Z%-l=R00nH_lR z0lXyR_?cb*QWO7DV1Qp=@hdJ-(L%@%4DRxKICWeAX?lh+cD|#$wR9T`{V+zUU6{yj zYdv|6Z1A#+Bt?6?4|Di) z63QgdWdY~wd^zvjA;S-2Mz;`^dMIMvaYBm46rQ8cOyz|zzTEE=WyB!DBoJ1LLaM{2 zJ-)!&H%-AwXKWdoOdy41*mBj_Xq1}v<=4)8)QoZY!NPl#YJcl8uzmHn2b!+}w6-3~ z&%;dX&r1-;!wiTMpb0?ErT}ubk-e4Y&kb=aQwkRciXYX7;-9yd{{L>V-Q}EhH4PO% z1YOhB5YB&^Y8^wM5PbwrJOL;|f82(1fVkK>xcC6s?SU#hJZSpyg81hW)<0z#HN6Lp z`Qsh2_WRuTy+T@PQ6L)uVj~}d9%uLR0K;EtxX~9S5Rv#ZJ-09$9Tw7ILM7_eM#u2UapPtINu;Y z9pZMd#ao*O*H7u%(Y=|xM;HX_?JHeqx*k%%wyv~~6^q&&*cRC&mhA9AV}EL1YA`kX zC~y|;W~oACQY@dvWj9R_Nkj5Fsy(#2>*&%maZh!Ne^+V;+#4D-++> z`{=8q-nqO^gqFwbo|T`RIA4`en|V&SnWE4G68Y!~Z zlT^ zW>I;suEf~K#wyJw!@@eG{BJKbkfSlPv#=l_h=8FD)qnc0z`%a9oe`^}vlZCR=saWeeqRHY7HX*&M&g;HoxLaAA1iD1zu#SH7W z!o9G~wk2k2XxR?@2Y9G4UXFtjxAKAysNG((Md-kUM$2Rv7( z&hN0U)^4xBiTKR!AbvGFj=zbEqA#d3VC}VAtl%c7U@`O2E`5Ws8P-j2@BJrKe%7hz z>m6R0NydnMVL=zpYKLag)D^`n*mu2H^8<{@cJHP_3F*5(G^ynXY39U?3E<&_nw4gv z%r2Ba%?YNlSD%RD3Ym*H1*9-evw( z*QJxK6%4E!sUD~dJejFWPI1ncj0Ye4-@Hhu{h~#g_tbhkEto*lSCr?=;S82%SZj|N zL>DYnN7CMM>`Ubm916-fkM-&EA-r;f1u{GT6OJjpnD!G?L{>92rqpSpiUec*lM|fQ z#NfKHuPa7`F?@5J)@31TRHw9P!Teu`PiaNACkC{@ck1B;X5I*m1_F#GFZVW>jxRm0 zIka?hsg0k!S2wRx@WTFTgZYAfwmlJY?b~4^#LTqJhmb&yQB&&0u~&F-xUA|-*MMfWJVkw^gHTA3W9(-@`a4+Pl=@jkJ1uoS#di3fG&y+*Hk`(NABX z;^QP?_2Q1@8OCBRrDDz^uJ8+akT-_t_^muqvE& zua3M~XBQovJUN@NN-H_tsUA%=+HHGoz0D|sDXI%IbMc)qwJxYAd+Wm46;T!h(QDX? zoT$`rgo={QZi=q?8Dh$-!Y$y!Bgm-Jo(sP6B2wuFRcrZ;Y3Jjz&HVsP9<@$B9F|(W zX7AbF1Eaj9Q$*@Fa}5~GoH zjcgI#d&O@_2IkT9dxYVSEyTO$S+}C?F_v?lry153-DcA#+gL0bI>)muUJ1DP+0FL> zR#^sngkP^MDeb$yKu`8ux2behMO{H0INgLcMd?4Km}b33SGMF*vU4?~8RIU&BDkbY zO*^dIYbZQzO09gIE^?F>WOsx$s<%h4egfZ9J+U31hMAX+h}pAYt>gC&Z6d8;gnf@eK#6+ z@lk~gTZ}YlFvOF8)+eC3uwHd>svZq)*Fkje%KlLOR(@49ZyWZ>UJt+SPUf7kErhO# zyaVF7M)VC7I3b-W<;LDF=)H(zrq;k)-@xT;ZKjm^FHn&fHBQ;iGnT9w$`#9M0<3yY zkU7ku>a1yaIkY4ZwOEs6tx(7mua9fk z)V{E9D|hFoWqzve-n(<~Rx9gbFhPg<(kQj~(oejm%}9QHMSs-YDy=U|zt3MeqT5^u ze!4{<01g@RD}0d9R#YyI{~O7d>B7N}(B>B188|e#XB&f|a^JFar(pGAy`-2Sp55A& zO|aTPXPIZ8TG*9jLq}#pFlRb8!5ewQ45t)3k zAA}20ti3sPY;+g%U2C9yNpIfkg%j|=IIv>-`YPQEb(%J96_l^%AEolVXr@oKBZ!yu zF|j+&is&1nE(E0NEQY5HYq95Y&NMWq41pRNdDK(ds1fa+eVrGyD|9{O3qH%XFrvj5JI?s0J#Hc;vL`L$Imu*0I`a z!0;`M#&9BH43zg_FH3RfjzIw!Djxx=u*>q+;a;2d^yPPaomcT};nGYo z{Tt2$OA{Yh7u$N}^itjnu`l6P4PXVG50s7Rdp(l6FNV?Uq~ngNW4lJ2&uq7@dE)fc z?`^gYc3={kVb_I=4Dl{{6@N&+NrQ0$YY<3JW`UudUwF0^j@W^2=__Hq=N0 zRVxwb`o5i6DE6(Tsx}GX-TWYTi!9|;8>8MqJ(5?M>p2H$MA$N4^fc_GhzemU$_La5 zR>M>S$W`l`RiBo)c!w{sC})Z%zYU{?n9rlmsTF-`_?S;V(z#<}1lL=4MbUO89r810 zxN?t272K;gr=(@sg0B21T95ODwlPgClzh%4qfY{4l?B|#k0Z{8obwU7;dDz5nM2A3 zWskkqz)u-m#|Ro8CkL5VHT7Owp>$=iMyY?2-9cIGJMWy{da9Vd-j$hTL5}~30Is|* z;qeUFBGIxOiD(;rWjoRQwuZ~bqbqKmUOH`H7ExJ%D9j%4ZtJ;kwtVyu*cRXv{X zJcz=ZqOk3gxLASnxbB~=#ZDwYAl$vT5izv>ikEBm?u|UH#pCj=BYYZy2_Mo*UWNPO z?}A>5xJ5Tr-};Tmzr39d0IxX6TGx?ntRN%V>kEfp_#!Y)Ld zt(siD`fR$_{UmP{lK^l*;L+xDuWE0n&9!tAzwn*H8xB?6qx@+U z!UaJRJCnd~CN)hHv4m%~=WRpsq;1NTy|1wzWu?$CpY5JMURUBvluCR`V+S=vW>aNr z8SIhW{-m7-LGEHXHtJ1ENL~Btxl9wKTU7^(OnZPrbmY*rUm!c!+)1Xt(^!eOe!JD1t*ZeuSe z`gJKla`GbIHP^R+%jCEM^Ed*hXzKT=rKOie7a<_2%FeN?*UCP`GgpW_`c7Oor?^H$ zm@U6@0&i7l(j?!6G}ah7ov(#_`09iDxbJj%1=*rk=t_vY+8|hqSuQ{Vc)aOi-u4dC zUL1^CzJoP|NCer7_|&MVR(9kpahKpxPq%hEfz(jt?6pkY-7{T^p>PXN%|S?$!w=TD z!P!nv&gXn<7(sCLe&$dVS?1TMAFFN)pM@CmlRfcujc?-j_NgJJymoXxdx_fLc(5V0Cw!eNAJkW*8Nch9@P4z2=%Zz8Hm zgsqwwi5UzSl@mrfFYnG!EE5fO5^cUR3c^nEy7Cs$eqk}Ujsdt2Qqd4ZDq(5Ay*4Ak zWhtebuEjjQi$FoWjJ~miZiH>ro6>|R-drP6B(=N?n!0;WdCR#Q!SR8^L1^mFEuTI& zqUJDDR~nAISZ$r@4b)UpRYJwnPBoSI1Y1^GBC!{znA4oPUTcrT*vntp`_cyfU>{o3 zD%}%}F~o!L$t`Z+LCV*w(zeeC)?ORaSXo^xPbD9hMf23)DrYY#4!o$x z>7XDNo$HUg=ZbM5$ldF_4LCwrL?zXLcWm|Vhbqh(3b7*# zEq1dHBz&B)Y42$Cn(|dlo`d5tDk~Ye?v>@|ox?zi;F)FUw=f%98YGy$ zok0%CU$;S&5U^!U=uilUL)+s4{<@2($UUR~r?s<=s`5|wHqz4FjdV+QhjdAIBi-HI z-6#cbHXc$phAazS2yJaQ|0mFlSFdxq+FFM#}`Td(}zJMyjX3I1)qLwc64+)&g zzp50PCT25*beW6~V|+lwQI-4^l|JZ^O+p7ke5sU!wrqF^dkThUbcMZja*g!bjvz&K zVgFK=D|pY_<3@kQ3q3>jJY4RB3a@!fP$HV${bjL^`amh!U+czf?I%htHUwNz$3R^3 zLpfS!Eh?PxWiG3|H>*SpW-u?^)&Y!v)~02j?CcdwE!LGtKBes!)0&{MLcy#Y z)1Xh>eejTR<-aVQD-qX{n!-$)nDaCbASZrOvn`UjDhS6Mf*$xgYHabec3C0*+B&`_ zRA{(r5o0Z@tQ;QDA@Y@^&iAm?eAh^awu~`sg$Q2UfQRIUX@Bd^y!NG^T90v$VPPtX zOVU6MRGP?7$)Aq(`0MFz`^q{5*1mC7bc?(%rE8-8+DVxk>}>FW8Mo+Avs^2l{q~q5G+-&xv7a?S$s2{?I8?+bE7b5pKP)zS z3*9?B#%Bv32r14QTNcaQF=og0_4YdOhlraT(vHE_gwOoKC+{q7drH3lkXo6cTlX}; zFznwyR&~KMGUIz;qSHO^NHILa`u5R(i$QU<)MunMO;@Y$sI9{dl>bMRLBdy~kGeiN z=(I8quHtKv>Vn&@$K84CxG`&J%@f)TvZ^%32u}rfWG@GTYyi2vaU| z{qB%YywVeX$bp{{93f%Rl5A1+aJZ$AE9>;f&E$=uUM$6Mt}4)n4z6;v7u{LPstK9M z#-+@vBbgHN!#pg8A`WM9=A!rU?e2g#+P-r>JLuQsKYwG|yhP}>iAk$Avqb{mkG>Mx zmIJ9KFr5?o-qi4of{G9j@%DTl?{3d;Vo(i-mI9vvy|mRv_+? zUZzqlo18F1GLQI*FR1U@7pQtSFp^d+^<7dXt=ptq3$9;n zq5ZSJmdv|64!7un?2h8ML$N0J;kO}lV~NJt&f-AQAr?&D9O1mVI~#48)zX!!yip8- z5!r!2iYri%a_E7{b(l@VE#zFuvnb%9CG72?Be{@e zHTLrqF4EhJ+AptxdEemOlHziYXR#6KDHk~lmfzrT_DXJpWSlFa^i_>T8f*pb-G{x) zV&wPU+%_yLHLsuwEgB->n z%CaTlN&+I4%8J&owl0k5jYA!_PyuFN?3sz83{KUk;p$`u7Z10m-RcMaLg^ra${W~JB$zzDV*DX- zJo6Jo;FNLmVM{c9>lmH>fxZPig>p;lgU#3cF^ha0c7_CX@p?WDD__#IL)s&kAMSOh z&O>=6r)&0M-4lwj70+oT+Mr+_C*?dRL0tiv#z7E0#V2HE@66SCg*z<|i1gT!U{jZ`CoFaZtwN1r~=gO*nt_7|(;U zk!V9N(rXNZYE#|oO28h`Mpe1nFr=V2cp)mQn4lCB`r0IU2w;slxuZ@>j}fgfzdhO` z2W!P3Sj2KV-o=u34m)bIat-1TbXCSMqUtNa@G{zk2rNiU{2<|AFZfsv4{|NMz!clK zu{QANyrcWuPP!NM?vQd~>d{uHaanX{D6NNSN%Y*;^K&EYt*s!Kc{cmHes5#I{TcS8 zF9Mg*rE|r5!_-kzG1?VZLpX9)n@B1sB*Ce4ws_>+(9zRN>35Qbi@^s;n&6Y7_KeG^ zpIdI3A~G4c_!>RJH6N_y*bnI5DD%c-E>KB;+@V+$Nzt-(ezf}qYd~Mz^#y!8|MK2J zl}i(|kvcC1F5epLsaWu0*Bnon1x<@OE_NK@gLl+X58m%UQx-jBSjYSzBtdMXBFQb{ zgsZP#rNhw`X%RFq-#Sxu(T7Io=?i{>J?HKfx>Qf0xq+$|=GuJV>GJ>$)U5sl}g@Bfb};!w)0X zfJnUAv6QiI>HcdK?I|ix3t|NVbz$Yv&gx?SPE-Giy9j-*6dgsY9lR{Sfc(1{01V zjkY7h&O%B2d&k(9CsMMhI&HxzG1U=Wr%5>Gw0&B#i=H_Q%!uEQS;LqWw~wegB!mUTmOlpxvw6D}(F0W} zs2Ef@`>V6a+9CzevBYGFCRLG^DoWniz?&1lCv6Yz!jnP);bZHKUr}g4n+ES?VTpbJ zE#tNt-`4aIVN{L|Gf*M9PH@KZbLu$Jh9p8|(r-5qNFBbKNiuyuIlSkGlgIrjsi6c~ zCuq&R{2(+^u5Uun_e!*np4wU9@hmjRp$9aFrk{S;-OCpy)uw7IZh@O?Rd#C(sBJDQ zqcfQzNWC#@RngJ&MH59LsiuLERG;82W-(7nFGd>UhdjO^jiaj!Dzx4{w-L5ry|DV+ z0OJ8QJD*!t3W0vSdMbU9ff!6S35P99zh6Y{sQZ;g40(?yb4G2ri(>_(Cm$44dhvd; z_p!x{xVNz=!pZT?lW?hBefByE1w*L%pa%3gIwSQRFA@p_k4<&@@|^2B8G|T95RB8O zaPoRln_((@8Px%zyqAL$u*@9_M5z175qS+L&jC`hBL!tjbJTCZhqfF^C7*p+%|6w< z`So@HOfk`Uoi33}YS1yaw5!ywKCh#pUA9t!6;%)QW0*`#zlbfL^7<@|?Esww`+*V- z`-du12Z_21XrFNc{@;iBSNUgCIRx!qmz(Dg4HX9AlF=xRbL(89wK)4!{VndYNv;H8 z{2yHA-eOXY()(&8ebGU{NnDhD3d2hQYmeF@@e1YFtYR>wZ>}DZbBD8D6W9(PmyuDVZ|YOd{h-%qpRQH@u?7yH3Z5 z-fT&BHE3!eBfu^R4!{2>TGzQ2X4v*qy_?>oHAZA7jE=t z4czDs&+2X&732Jq)#$g)rpMIwI`?oEu(>(qm25;{h>6rm%ymx~LHvc|6tkZ-G|KOr z4;1<<3(WDK8?bbGF@C&JUKBHolR#zFE}gL0)!Yo4L{_f%H!P>RRxXBqtrcYn`an8)uG7q_@@#bX3T`EET_>qB>IEWFoZcY6}$w{oRIQkHg25reM6@1 zckP(4-<=4>Q?SrK2)pZ4;-vfU$x`g&qhmo8F{*tmg?SqbQ{YyD?x7$%+snIvtno(G z!l{A%K>CA|ZHaTfL8SrbXqJUaAB;ow^|+uWbJg1LmmMb>IIA+SJU%AHz23m`(M#ru z_=sTGU{Oluayf_o3RAL9Gor_yZi$i*N6FJ4ikCX{#;0W89yU-c2j!7v&wY%@5p+aU zbX>?(=GgBN%`m1+X!?klSGc90MFG>Hr7_o|Or+Mf8$e-uWX9Z2{7BHDblpFrVbJCGmaeuUD!D{#Fh z{(jnwib7N3zy$la?-R~CFL9S%!;=*aLDqtX`(>y-y3>y zVatqOwi0D^?-5&>^qR_f=xV<|oh@-FH^lJKVpO>YO3(XBr@F36CJD{4<5-KS;)r`T z9y=dNgmo-YmRRwq^8Oxa`-#PH?`#s8;>WENyi8D`6lqfU4ONCsE`xJx_K<0=dBLzO z!6tis{yTeU6!)XMCwj%qh={hfQj0g*cc{+ranf~Plr`}~Gq~VSSReHda?Fdo&)j(O z?UtA|(smftag!fc5(MDeHvA1~BpYP}FdQ+2+l%Qr=hrrRirhQ=EG^oJHdY}dEJ0jvV;m6-eHQyod>kHGs2%;k;mn>@S%3zh~$OmY+47@QQp#% zP50)#r(M6+vzmsb>-B9k{1IVY?(U>hT{4W(!(cAjAF_U}ZJ>+?W_Q4Ad(>AL;x1F) z!-f~$ZxZ6s?QK@CvKM5G)9mVcAr6Tvb7hyl2Eki5!OeQ zF!HWlw&-@cRKXXmwN$^W^XYT&8?pL&grHu_;7bEg8M4=q&z6DYoWcEp1l_7*^yB~vgDYUSIdaI*&yaZhrKYNT~IUF(Mvbkh1 zZ7uO5=Y@G=PbKnL@v_LBXQmpjJU8wA*J|v)(qW5D@LLM=q9SWIc{NkcU$Q)qn@RGL zn~zdy535u*@PTp|bGD;Y-MxuFgQ@_3C+LZm4eq2to|=qrWmv57ZBj;Yw?3HIf^F`l zZy20wrNtp$*#tJX+Zr{zv^~Zq%0;(wqbEBQ?b|e$X>{`!-YI?A(8*%}Eib|~qU8LmT+dBi2R(vvdo0le=*zkA__cos~7Hp^J`Fr7EYTK$YzuXZFZ}a>CD4Z5*9$>~+$AY#k%fM%C836|K0Q z(N^D4CeJhXtAe55zq8S+7ehIBUx`q9v=Q;vgky&BQBu0x^chl&(Y}ZnH4T-RrP*!} z0jKU%6Gh!BWZ}>2Hz|8wq>c{9bMi_DrwYVl8O$xE?b6i~qjMKKe-{2~#?1M?i!YLG zt9ll`s94Bl*lr*jY4(_BZlX=wQh>=_$%v#6h1SL{QqcJADwofJLWsLS3~xbScTyR7 zB}l&26jDc|(s;=DVR7QGLCITZ>g&1OhC{i&&pDn1Dpq8=2T$>*-=3_NGtCsZ&k^8a zsK!h#bbFg`#jgu^G-Dr4e?c8SmDz6TNRu8lyTY_+Xdm*>7--mxskVPi8tSejv}5z# z1+Qhmxm`A4_2=JpwhL(z6hEN2==+^ZnFvPi_QxhNO3wk5>{D_6(S_oqj`2SIXfcdm_miX<2*%o3#X{0ZzHqb22stv z*V8WV<8oLacg4&+J7Tx#Gxg*#z>HGtsizu^5P~cV#hI@WL$uf?QijIO6KT{vASsrs0i5OnSAoPdZB)TSnlIiM=KHs;+oP@2u?%_alpZKY*>Sl= z6dT@yCXI#NhzqY|0uf~HFYa2`U|Rgv;_ZCC6?7InpA0gK)e)wK?2NpZ+8o-6 z62cd``2r~)r02zjhe3Yhc_ZHxd!H^R_L=My>bKG;nZ~v;W2M)|%fj{;>qHdJNw!Mj zsYQh8yYeUQw8NAmXs?rk~@e^rrySONURy%QKq|Xgo`E!8P$Vv-gs#A7Egd>AT<5f8$@nLY;sj{VqEdcW8Zh!(S@a~`FPip1fnibS(9T= zmCU#e(P4_*He{b=CDjGfKXZXzgMN02?mgs)?RT3*qktAmMb7R<0x@8OL$O>aoulvP zWgfgha~_ObH|h&LA+LxA-hfQ4vz@?TL^84vT49ui@{?Mri+Hyz@tkcF=AvS_R*2{k zMPDhc60IgQN@9ew#sIEQ&Qsv)&ksV%kv0nLBwk3ScjfUDhbZ&vHE38`(8>dtVW}mP z%9*y!VS3-#AJikYvGXX!j1#zA)nQ?f2?sly64fQywrJ_l`A8j0hb;~xXhp!azWGDe zD1Ik5V39o<5<@zY>JJpk4{nXpd)wX^)CAh2cn1Cz<`a2D$`gMTO-QMf%mh$CoW)cy{Ve5fT-DALUtrLIXz=g@o`TP8CfvLhE?4uGnf_~Qx`MH_pP=JP(shisPUU-@_@SKs?34m!cd zU^U$I3z|1cqNz+Vgeo!g%Y%Nik~U`88)u0;Sk{d^!ad6U{-HfX5}m;FG@`xlq#u-2 zia|wn+R)2(CYv>lV_xd!PpQY@7CP7wxoTE1W{dIsyF(pC)j9#q?U5qb;9a;OS}r4q z?V!UUqs+zGE*bxU=I?Xc`MCHzxUlKC%Z;nz|i0H zR5Fc0bcJDquyCWlg~vkE+d~c0B$q{BTdKDHR7^!qLPI_67^FtxS-S9zW*witczl;$ z;8^^Zck>l>{Ms>+?viTgk?5`u#6TWgcWyyKNc*;`=!q80DL2-p`N+*&lQr!v(XXOP zUT0&Z-LGQ;M5ItNCv1)9TJTe=Imnbu$dCw(0x0i(botz?7F&Dqf3X-G1EcVtdmE4g z;ZdH9JGlPYGFYBdk*-z;ffqj9ooSuO$(D1(jskt8KV9GRZXCSF%OmqmdRlNih+HCv zt22{>1AA~63G^v95>-d|A)t4CgwIC5Bev`lWN)^}Lj$Pdgd ztsrNOp~Jvu3=xCIo@J2inW}p(@P|}uT5nvUKXAOeWgX>TQb{4Uq-vAVY4!evV`hFj zQKX2_V2`2A#~xRG2?@)IBUt1g=jh+60ZT5o^@$hkNFq;oXD^sZxnEqF5} z-95zxb4vHS{i=ApisA5z7G$|#MoNon8F}f5;yl0D%uc)lNbK!BkJ*X~J%5Pb4QniI z-J=k;-1~sc<#w!~^83syPp8sUSjdU#wv~8EmrGEM;K=MXZWs)RL3N6f@Qbj5G1QW} z2jb3eiO^xO+)#fgu&F(&PS-p39jJ{vx0-bJ^GZVlW*uw3;C!{Zb$bb);|0W=c(0@^E z-@Fm(NT}ZCZ||W`!J-C-&4LaZ4T?drM(i1xM}L`-XaDJz)RUN&w6<5z3cuoMD2twH zdY-o@&&YZ`gx8|PYm99(p1IdF&qrew*#2ksKYehfV4pPuqc5 zJsLYkKbJATDqO%?eZg$lGWzj^1jrM9bmIp{gw;AuZd<(WC=@b%9G=EwFaoXmSV0EK zcQMY5h6)jrf-V&Y_||Tj9WWxIYAqEILZgIZM>`kdPRzZjVPN-f;ORBGJd^iuI^=S4 zl$O739R*E`Se9GE&;Mlev`s>)qgvIDuL!gKqQ2#MoFT6p5J}Ggf{7z)sHDI|8TAIu z$IyP8?YLHKf(c$`D*^^TRGJU-Z8r0jHxJ|Z`H+3)jPCY{WE{4eVl(f_nao#)RBRc! zpdQez%{K&ML+9~WTRqjw#cF<$akkM27$@lc)LYt5OU^!_JnZgRLV9ku#&`s-u(Xih-k6nURJ(RP^9Xw zgpYS4vDZUUUsX}}mL^NB0+L7=;pPwgsH|71OZ4InPBGM-b8?PJ;7(K*$}2g&R>%(V z6cE+*bEw+c!0(4zC~Mk2EntT!qWbBb!HRDZ++zs_x%dh1aT5ja=Iqw%ebF)zE|WL< zRuQq)((oali8_e*M&0)uy$&w+n_f*v?m~!+Q1|kH?FE|*ep6$-i7RK?M`u6o>wzS= z_|AHSDt6(4>a}M2=65W0A~dHK)lBtDU&a_ey+@+7H$zzud}<7Pke~PP@;;o!$jF07 zRUlyO`}~8(%Sz1d^J(5MEs`_Rtg1>aD7tH=6otK6@{3@9?`J_%yq3Bp7qF6Aw$NV4 zaS8Sq-F?aLN0E{*?opouy@fXYqHIqCj{~}zZcfpF! zU&8N|ofSgv*2kU->2((2Px5kF5@Dugz7bsIu|X+2Yaci+FRmcb#$3L~pF?3%P|5E8 z!HZ1@X?y#NP}sd?$I|B;u7UONCtUy8j{`}dr!D&>(gST6WDBsYl>_6`^P`bzlwQBT za_Ifon+QtYQP{i{T{5{lO`%np5=_D4pZJG0#(d#nFEy)Z9N)9IpndWHtGx@+NTF^+v7P*&zkSLwYxvt@)|#IM zLxl8W(Fr%TFNH58{$a={hpW2_GO;|5gAt?Qz1-vd4}2J-bMw)6xk{Fp>yN8U1|cog zIE+{=iB$&r!OW-&n2z|>QY@~Tx;cb)H0&PnX8y)_vI-n1p>@}?pDX7A1RERItt{`j^JIc-21CQJN&afQAJ!_2{-d1*D&8)E- zUu3sv&A0U8D7())ADvcK@Tflw&c)A^c#BD|64C0(Lk@nsk%q(%ejehMTR*3-f}THe zSsce+$hB&$wK;0AAi+qm{@sisNVkSmZ@5tVBagLY_onp#hrqC7_`npoAZ?f{;u8OF%Mum}nungPi0>IW-SQv_7 zuB{tiM?e=VVoJ_sLZpcW)4({)wc?--co}!_>o>0#qeqt~DG^tjzxnqdN%cLm+fp`d zsBf80y2^H};~;oZ;r{5^;JMzrZYLiLDA%m{%_H70w=yxVP?p~7Jy(a=D6H=b%$OPi!rw$a^CBL_g*PC4VL*}MklZxAu zw9Fg59j!?|-sMB=5_R`+aSbV;NNpx`SLAfNGM+9s&C;X7S;fxFfwepdOCqC@nPwe& zL!gLw9KTZx-**n>?=0upu640#hfgb*_BZO#Rw1GWOA;)o!djb_a;rC4x=|xinh^r5!6hocv|HRc3>aJmpPd52*^vfX5U|D{)wxt=?&c{xpU_d2AhX)7u4W#R* z_l$Tawpdnt6|!4C&<7S+u5RPPr8Cg%{P>v!Rl|i2I(eBv%QPq-980$vR{Nve4!&lR z$3tG=R$3*xM)Z55L&m(@ESecdj1CSDe>btmSx-3#*LoG+^*zEu?QDxpNAz4nBPsZ# zF#v>EhPf1^%Xx|;tn+)tP4AgF+@q(@nfwBCly}sDSeB$IaYNiJ9Cb0yeUkBJE~P&T zQy)@stVQv?NqvB~DI1d`uV6QCI5Gm1Ao@qRTJD1BMwP^Bg!kp}`w&v>-X~)oywwUJ zOW9}I`ix{P@bw)hyO~8}ATlz{`8EQ2M8R`m`sE$)J*JYVUS6fFC+?I{GyJ>CLI3Uc0L)HF7G+MM+h&gLgp( zdoBmYFz4^ycXVccYIj8zvTXI5Xz7(*J94#UADU#h6$)nx`XpjC$M9pj<40F=2!UTZ z(aMCbe%)0Se>%y%XYvkRX+(ny?5~oEK~(uFBLy5bw$e>7Rd02LH2wQhT+16>(*UXVdshEit z8Sg5df0M6lj7ek=A%@Xz&3&q2}Nxh&JcU>it_a<^Xjp&$$XWjGm6uH@FCRc&juE2CY)xMdCjX0R zfFku>~|K+rQo^SkzRP&{5?f<8x2Ydj40^irj@R#cabR@7kU%+@k+w5z+77Wnw zf89L`hyy5veZ>)h1sVrfMJ!-AphWdGJPr=n@c%2NDj*x6+w_$!`Q@exP*0OTGo1ng z06ITk0iF?n2KXZbC?El#m-Cfi6%lBH|2s=3AReG8^A+#lWyurBuMqIwOu%?RZ{utH z^2-rwAmf3yI0B{vDga;85zzl2{Vxas0Z9P0ey=2be?syfOa1`40L6Q+TrC*DULl|~ ze1O4#KD^i9OiW;d|J05L$N*@Idu0H{29n_)ZnOU~FaEQ~^6wQ+0E$t-bU>HdYr4Tp zlL1iQ8vjWDua1-d;c5WFtOf`K=rDT)^1%HApub`?14IE7alN7lzU-s}d2Rkk%LPaP zXw!Nncmkf_ue!AWVF0~UuP}myz+R5OW~l-s0aQ!9l0XvyO#-}V3NRkfzw{b!Kn!gB zAJ~`x;{nA%ukneY=M7*_^hE}(7VHMfQi=-j{To&cl) z^h3PTM8B+W0d*CCu|)vJ0@@T_V;2~Jj{OhZ3V=L-c7azORVE;L{zKOQAQ51j|0|K( zdtixx?e+)E25ilL&5nAh^#bNe@y8wdfCzx)_OA%$EPsITXI1xr5P%)@uMpy_Ktudl zBRwDjU~l{@0>;Y{8gO3a{=qnUvnGTfzADsa&*9az{2p?d~{A=^Z%$m91sAo zX!{jFhYLu6|D<*s5C^b-`W0uM8)zJ0+o%D<0sEp~!}EE74*x6d(SRU;4bQJ2?!3T) z{MEi^Ko-Dy=2w;ve1E_KWLYy{I$-1PYr3PrpQQgU{l9>*fMv9=u|tAD#{yB23w*A; z&>!XgFO|4}xqzj&ua9>T_}o9Jzy-_)?2~=Xe-sAxasSnJSwI%RD%e++aSqY?s0Bc2G15CvKO~C&s83mjN*yHdzPgLXI%=<^H1K=pY zlNGO{o<9BSQUCs&1>kJJ)%(}koL0cj{`(?6;9$U|`PadVKo167p$E(W+#ddC#v5ik eVB!C3msnm3;-%*JMRxh}D~13Bgv;k|fBJuVj{!#j literal 0 HcmV?d00001 diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..a4e218ee2fac4ef1cd3ebbeddd774e5596707a85 GIT binary patch literal 18851 zcmXVWV|X2H*LK*(Y0}uX+1R#i+s2OF#%`QOO=C2+)3~v1H{99t<$m68e$FvJuH%|@ zYOO^YkAML6{=fi^cAi#_j!q`7?k>K5CRV})KaX6{fIL2KS?vM`Rb|vSwFZ zewab!Vkxzi&*(eXrJ7l)x@`B-`nP~~?8(o0nE!gPSZD$>ec!tFEBNPpuz#Q@n_wFr z_8>;V;HH0X#H&I>M|O@9O;z&<-I-SA{5|GoHT43@42=?;wL3E zP4g<#yo&$Ny5&lue0h!g-|=7OY+stmUi*}KnU!mqr83?kCCuN0p3Y<6{e?AMYhmWS z9c{!Fdndsy8N=n_w)Y#!P9MHl_%!uC=YEmnXenpA`yjC%S)sONo~OI;eG#hvi6v0V z`Fxu|lmQkg=^&vD8F}PWcm67;!F|ZzmB}{(uJ&uQ4_~HevfDC*hI`VK9J+t!tU|m# ztA;6=FvIvB?@4^$?GJqE!-TVuim889hf;jpzlMI(vTTIc0Av>25&Qi*Fj#Mfu7F_jSnx!w{UBL zLH^1QV=Iv7gW0~++bG zSpRbs5`g!{_brj=d;Xhy@crQhwO@dL;FpN#+9D)}ZC~KnD3$24XAQ(@ z*{ocYen)__S3(H_2@J|CZwZ5aT>U)#!{)zy5o!~RE>Oxkc(&5~cO`>6K(itu(k>d( zDJ&ASR89V+Xqess?lcd@j}gJX*c2{q`|1&QphC?3%eqKZ3rih`GCR|H)duog!QnM< zw*%SfFmtdDGJtGt+WCA2_iSBwF+h4^r@MzL@n`U-V24?39391)M){g?D18$W17Ei= zVDq3`$<1$)+W1~9m6OPbnFP_Z#<7mu&JhDt>wabctZkgLqdZriY;%zJ zKP7SG@x->ehKxgA&T4haKt{Q1ch~9q4!N}kOuFG;GYs}%f8Hd6vAVCDo!DP4`hN?d zA)1DJAQ^7qLq}Q@?}&zet@OA{wLlYV>O!?bO;fQh>x$q^e=FM2`r#uZ6EK^?iCam=(Xa{oTC2+AZ2-&*0(7fG}7G7K1hw|1E zWu2{46M)JF++|s^^|eHz;9Ud3Q@+i0rX7u#Uj4jUZ*fALjz5)IqJizW-LcobJsOY@ zXAfSdY#vd5_sF5>UqL9L(|buQu?a~vd`D|%f3mXK%AHvdGhI?N|FLrB0Plc@8eP^j zinU!us*a4ycAoF0m0&>|P7_VybuDf=ZEeo@U^TqSK%oDrz$d#Mlg|vsBwd8YhL=48 z4)-mR*%+|p>HfrG2%5c1uUCnzV|}l89y2~)D>REEe;Nom;(DBY6N+HA^*4nV{n-WO zg(q5O1UC%+9GPQtBy;sJ4I&2-Lc|H?!j`L?-rBSon;Qxvq_VRIbTXId{FB=TODSZ# zkdcFU%IgB*8a&2Vrk1rb2Q8%RFkkdLtFm-qT5U#oHg}9Q)FxU@Xm(6p8ZijisSA-B znH=L-g7GA2M1Kugz&%i_`kpE&jzD-RBN#A^PLya^(!~!KKGdrk)Wl@Y^ulFj$yv%2 z2HvhM(Q+J0@sbKhegB~K$dxF^yWrfMj!ngYA{wnR{1LPo_*>hdyS@2KgN8PZ^VQ`+{H73>ChJa~d)2py~i#=i2FG&0US->aYs zQgaCUHp_+Pc=-t6f&sF>)Wp1`Nyt>xAJi{#EmKO$WkFLX^jtyeXgIWL?IQ)C5y=Q-^1 zah{hZ^R<_Kc+rJQn=rS6&817#u2!UQXRvmjw&k^JwfRk$v79fWF-@{Y$#V3&pyKnL z97Lg2uE;@7Et#Z@x5n}esU?6hWeESjQMW3RU8Lhtb~ zCY9~6t^yJw0Y{_YFN7tmkyaVnu}Hhw`nIeOogvO+G^+ag50h|pD6w7`Pq1Pw1VvE$ z(B$JS95;`X=6H6wE7n|#E#9T-A_^!EFtfhr%3#~huqmrGap#SA(@f#mGk6XsPxFz)S~}FP_X>BvB^%*M(AXwY%pxy90dx}O45F`He3)MBG4Pz z!Mf`TD!5=j#5mr$>Pg(Uqd8u>GL?w?qsnl;B2P%`w40Z1tc(h`^W?=kcN45IGuWqT zkxZ)bWTR)XWd3q#V_K#0kTagGKbwsZE`ySCyL`tW!bM7qfIHI*wdDG1Y6;*k0{6D( z`jkjS8y8`M6iuvJ=UAR8J)UW`A~tx0UGzKcl1CUeEe|^5oKAc4hex!hNj-H*5s|8; zPv$XgMmc3R&M8POAokyW6xB=3j7){nz7NI;(F6>0IAO$$!ps{N7pHFhpuilHPJ-K3 zgYm#(J?+%iv6B<jtDZ*DNco3(OPHU0Q(C*&mb zHj({&z|75j8k8%{u^U{H)ib3`0jZt8^cLu$G^u*(aemMou1vHZpz=dkUbKX0Q-TI_#Mtf(+PNTV`)RpyGJR4`cij9lHPGgcK8 z;jiGvt3awi<&UY*BB`>oYusEuLu9akzC}AhqG}11M9%n~l2ojza;_XJd&ViwXe;5- zmvenq=}mZ}$}^5s26ELhe+w!%83PEDfq?MlF`y&Q=0dsk51iQgv6<_Kx(Rtw-_BbQ z`D*+v$m@fE+;h*`X5~JBItIMXi|Rt|A4W^>r!SI6uD0@$WSv!p%&^9jpOqeY=-sIrUHOvMcy$`=G^wc`j|D>47`!lxEhDtZSxA($cI zKFPupiFxeQRD@vM0hqR|+d$rw@+~BBVbF^-i^eAF-oy9XMH-Cq+UxlWOER#mU__3ja6rj>poOa$t z(UuO*Oaa>Mo?TSOK)K04yF)6_eqochAvSF~X6>8eA|ZdNhZvHm0b!I3C;~>D1*`yk z6o%)k&}EQ1AH4aQ(ueVQjyY{}(IFuHj`S;Vr2Kxqe&FL-BY?6E5_mJRM%Zo>5&Kh@ zK!{!ypM>akJJ3&Y-P?QPw^$+nw%vN$~x- z*ibXQo4mz$oJU;_Gf@nv-R-$b1ZtTfg!p}TQ+Jo&un;Ei!%y#z>hA!UCfp&=<2;1) zLhStvLx93Cs06>z5bWcue9i#rI2O-H+2JxnpWvH`uyL7qIu}fu|nh_AlU~IxfhV> zLk~3zt+jSHlyo1-VSxB&82Y|Ed}M- zL&Vj^S!0}Z=i6wUbTh-d!rGd{{lV^D^}jPaD$F@XZ=2Ypo5q>r<&j@^{`EKuTmUOI zw;)Gp;H-J60zl>O+VmK>UFW03Kpy#^ePa5Bjzej!g!2?Go#koyzUTe!o41aBJCOf_6dM0@d)}qTFhbzCcpu7I6&`r+5E!y$odo{MZ2|K?Z$W}s5K@!S{N2|; ztEUCnHeo?rcDwc@j1+(^@p^;&nIRmC+ zF`YIxETwXg5x5!$0))Vzna!aRQw)SJgn!;&k%j`9q5M%5J5n$-hG~d z>1W7YBJo?f8f*91HQ$GR;TlKza7}#T^ywnI;pbs_5)Q)bKux?3nN7@DN*m$s*tm4^ z@#(ier1;Gfw${dAWhwtrN?c6$=2Sb+Jfm7wdo*TkA_EtT%kD=6o|U+;@24XmPuU9s zG6DCrc34BEKSLX7cp$yndn(BkB-AUF<8+{7KEs-Xr?YRn-2g;QxHmwsOVT*7P;(D_ zT>+TmT{|(Nc>8apVKSj)$OAi4$X-Lp=t*Q-AjkPE!`~=bI$eeNF2COGPfzfU)?BA# zW{<1r*sAF@F5DTKWjt|Ko&&i%z=iQD#N-4Z+y(NuZvmS(n>?FjikMI_rWNt;1dCZp z)>=@#7JC4^Eo%aB*);zM@?nJZ&_hgIA)xE1EiPEWG4+sc$QbJ@z^z!Qc~KCk|(5ORY^ zyJQ-A=crM~6SQxBWhXp5o4?zc%}rf5OS#)Q+j+j;H}zGEh@GTSR)Ojsk?1`hQq?Wm zo_%eUki6}K4D~^F>bTg{38_#cDn?6k1MQK^X-TpDJX5EsIPAzuv)<{q}hegHyPZ>%GL zLCdAKV&^XKa~1T_4zl&@39vrz>1w_Kj`D(SW=Qdn3kY-`2J7;A@jCMTkI?6`<*|Gf z!9AZYhMs`{JK%}|@>+BaNM0%$(+CPG?H5TBlXATVxj#>=Bz5xpdw9C9R99-fn;-?P ziE(MYZg*+%aSqf9M4E|(nZqn54fb?vb)CxZk^JfH|MFcQX9`;%4(b*5 zFSW({#u2u@Trxa^E^gmpd;{4BybeG_o~wDmJ~LOen9#jt4ad0T5FPJc8Cn3hq-zGU zv?aF!cr(6kmF)w&g7eAns(mgTv!0H$j-w~URgQnKf!a$Tunr7zdIX{I2bsccKtkBV z#`^D>(6OL<+JU>||Ed27sO$<@|8*Qumn2d6?Ip36^iB`-Ug>!A1Nf6R!8y-0=8Wr*r5;^cQGljmW;c`qdDH3ulC3CpF=`b?msy;TZ zOz6?>7BhWC=CkQ=fuo}h)VF(892G63XR<1FAuY6>7A z@b`I3A{p>_1D3T`&NqLnni@vi5Gas_+}>9)+q@_-2PW6Ethf&WSL}d6>p0;4A3wB! zYd7AJ86KQS3!{b^Vj-x@Ao%+h)}QLX0*i!*0=k=JL*JE)jswQaa!iO?`WMA22+6FH@J3@H3LTD3tv@%E)2gJ<2|V5dY&CvGM|b z;Ot!rG{B@+Al6jSDRG{W+H_CLk$Aw=xHvX0%941Nmu+RBjyuRtuC}2=fr>9~)7p~9 ze`qs08pHkMOKkF}?!B!$o))-g7jSqd7MqRBZM{c5ehl9O==+%eTp03`NCog8iol?E zUl31hyU9^aLZ+_uh!4NZ4n3Up3NO%WDU^agZsOJDXQp2ER}?GSzAz(u8WH0G{;NmD{ntJpKt-geeleoJaG{r?UybSqU$beGP^umz|HY_F zC0Y5GBFKDz>5Su!Z3^W5?z6i0dm!z%BA1?GiKL!u0&)@|W4ql(1^mhJY#zSg@+EK7(3y}$d??0cdETir7qDOrkO;$Zq&KUrUbimCC+_QGl z*Apw*tHrf!aRf-&vfcy7Wp_O&-tVnaa)?O^G2@Pg7R9m5ZRKr`*7Mfl0N`^&0HylD zrzC!xO(UG$y*xf!!wCGU`@es91Uf5wey4!vzH8vT!=ohkptO|fhPkIFC`f$_1{+}A z0g7UQ>U-eNHE?-J@C-tqc@#8<^W8q2{)1r~8hUmM`#bJgL>yxB9(Z0suN`)8JzGDY zfT$;tRp7ajtMLczRB8IL3MCuuhZjV=ENl{1|3jHG%8+6K&Ovi_a}7{-%K(|Ggk;d6 zcb(yy8eRqYuJA-lm!o2}G}WA151r6A_x!%7`h2?8SE6Vlj{udu(Y7ik5Q3Eu*TP29 zU+Yyf2_x38=d(12W2>~HDmY-#)j zUJv`8V*ArH9sHA7zL(}CsMM8OOr3u{<5Eu0IYw^kG{^)K?o7S{KJSb`xC3-SR-b@I zWuLmW9bB*gdp;Yyk)?}&tJ1!_ZU-lCu-E@(kxt-eUK|i@rFR3M-T=E# z0B6*SJFiWQR?@%r1uGLXLiF=8Rz#Guj#+U#t_Y(SO$li+37A_W6><`3u+QpyXTI`X z%W~ zWsnZPgQUpBQKT*p(X3-(i<0P9L{T~%OR_g-{fH%d^eT>K1#y0g2)O9M{^{^IqJE}i z$?_?&Bz^*U?l&F@Iy#G7^nZf|nWWkP&cz=)i8UvajyJB*1>m=f>%h^DL!nMGmHVSl zDx;7watyN%W@UC8uV=aND+BQcA*j~g|6sg@%HMBRITeNgK^^^egejJZK1ufe@AB$o_! z6m@g?`Z6x1^6cX;{R*K*Zlrp7Bz3e-2|3YMnD&uM?mfz5dU;Oz0f%F5L0wN_a;gF6?vP& z6?GbQrYpabUr31LvOu!zsh;nj-KraU-m@Nd8dmDzljlw4$;XMb#5fvvh z&MVvi-d@18#10S%0XU|9#slvRCJ2zem(1fZ@X=u(m|mOy7-S4p8y1R<%@I1j54`IE z^s3Gdkp2dA>;Oocj1ZF>Kpp_t|GnGep2tprNgi(4-DU9Q(omSGNmjK?*+p0p4Y;>XyjXeRE*3>${ z%g5*cdsrgC$4+@+TSU_1A+j6ipG>}x%s%7NdV^+wdN9fJ!oBgW&ol%_OC=$93lqi(HW!$2yF5BblGW{lumXG#2wrXAi z(H{!D(}9!CbCC4Gm8XqWVA6!;M8c0-ak>qmy8Tabcy`ILW8fh#Q`VF<7?*c4>}U z{W{dZMhSRHZUXcs4_Qk9FX?51m(B|DX7*wWWe$&X2q>5E*V4(a)&3_W9A~Q5QomMEmv|eE4xQO= zAmVy=nuKexRPS)0y0|VKtn(cRYE1zy6(DE7CICVem{}~Y?Qf!jc1Y0J43j)>qKK_dFX-3H;#f<i>M2n-Cr;>`FyZ@n$4j>g0+;X z!qb|GhifNb#*#wFLBQBnlL9Q-18Z_4fR{JpvkT>|iH(obtx0Xy+J#TYF2xsLUZj#R z2D>Ya3D7%Jbpy2NKy8=C0kD+@_E~xo!9Hp?c&!2lU<1?5fM*pL_%D%8FAW2sh)rv6 zBq9j66;gS@bNC2eBX=d6>Xnj*Am!WHbvip>%}nn5X|N&ofBJ19162bVSDt zDR{W|QQoTSc~GVSP$%w;5CHWFWMfl44*0w~zB|XSU_dXRe2w=Z6yx{ipzhOQg!Uy# zVgZ0#*cAdi`k#P-iwEcTw#pz7|0a?J=lXwXY@q8CJd&V@VWl(&!!U+iD%OnenhWtUu$}*UYyTU& z&6H5S&n3@4%d61mI-f91F5$A7;`z$s{3ZZd+$!>kx;lMJC}DI~i%h?%4*Pha0J=n;U4o8)aOGCiEY2Fm0%fOYf(#)N9hhy4OaSSf zY9L#S`@pi3&h1vze$T}F63KPJ$tV8YkF8YFMnmWSi+p&Ork?+#$pG@GeB@WYbmHAc`TP2NUx)~uxj?Esns4=g)?W$?Do?!y z1{g!%^%x%lI&Sj__IC9R_RkRH_+o*FU5-K2(K4irG6>HFKMSk==6N2XOY3!f@BZ|G zp&U({-)h75eov8aVsyZ6p(IS9WAM1+eRxJ%Vd=G&VCE_!!>9!|!3J7s-Y3A`?cv6) z@!z(ob{3>yGTGkuUc6|vkU~b@>u=VW2xv`h07xVV zQiXFs=E8dl{_pBmk|0|KpMj6`HS-rDgxB$=vm${z8i^mlI^Bhd8UOWS)W9aZ#K0Db z=5xV&I8Yb{P>IikOj`_(O`6cE>UtQ?uP~zC#&XS!FhKITe0{;=(uVX?DkpNQRR{cO zj}OEzf6$K=?%VX43g%3V@l1+awJ4qzx-*bB6})>Ee|ITbGVb^j@s~(RuNzNcR`mZ1 z)_i7tu~*!fEypSJ%c%qoUO@LLkb-dI@>^hily*IJA3jE?hE!1ysa>yYHS8~B=?X-; zKb^gj6VM{UIk5(6cm$MNw-nZajf->bXA|B#=tZZO%YH&Nj-1s6#8DTrXKOWhah%Px z?BE?#FJa!(=+{EVOB0l@k4Gb9H^IOD(kvA9;ux|vinVyQJg;3mI}tkw1<4yTS0kd1 zYPh5A!tu4(cyFh&H(Bue)~H%gz#|S}8t2&ks*YxSAe<&*ftmKE*=N1}$BILo8*ewG z%jEd8z8U_&`Vcofle)IN2EQ}H=6#Dv9lf4xm{hr(Ytec%dBFIC$0s}22o#*!^Vt99 z>qook3y&s2OMdw1Yx=Ir3?G1$!CI*M>stIIiG{ff{2}{#q=>f&mzbAW>#N`ID@nJR z*?caetMeHPbANvw6DAW?%1kt|*k61srj#37O(94ZNSQ{bm>lP$zzY@$q^c9Np2@93N3lap`j|s_4yT2*jn2$cvruAg_|aAINT-_d80}?Nr;SBNeYnFK zIZfXjRVcjCPY&_%R+c3S^Z1?tC#Xo`eqVWW1=7LT(!C&Jn@pC?HFdoO>f$-%k*JHG zI8TnsW-^UIS;s^89f1KUBMr0TPyk9nJq7!@M0#6HX@fb(;7$?24spL;(-WOo0^4Ee zydGOde-(b^k}k=A_UkvuC^B!le5kh{4JlS9c+<6%jq@|lvlPQiBP4EzZv#~>UxWb zWrUBX1)f-V9;bgCw{*(DDcHKCr~gI)>Ek5@w3~5UTQeCY{*!C!hm%=)2?q~BZ=B%yZ`3gqH&94ep z%u&DH9uArOlRr{!q{xRU7+5P<^F<^FVD016g74^8`RaPUjV2@10=sQ;$5SR1h?9kDreGAEa4c9`{DE2WIvpcpm6 z=J9agH}nO7C+Y9ST#XB-d+lKOfvYS|})-)FiJI_H^$Ntj7cX(MQZb2kNeiT#o&&CR<-msYT9;!|)dy%y07kw&YEF z_uAMX)JbC}7W<5}+{vXsK4PRg2v=8SZd$tbNqxPM`q8SNB!ytKD)^Yec(l~;h z7)z}Ba7Q>NMbs)qt&xlMhk*T7eo%xnA=IS*dFGs0LBf+wE98bTVUJUE{2vN=Zd$I( zphQ}tbddGM2vWVPdFuG-Un{@Y=gJ3en~%+^GSdX9)r}tX+rN+*Q?fTK*f0y%ECMi7 zfYOwz_2f|;eK>nLEIM72Vx-1D^3Atm8%o6ONb^pg zq_rdT1Boteq*fCX3&q6YUSNBt=7uriX1*1IV?^R;(H_1ms87=W6W#Qpv%>JzNi$8o zhBYt}=P5TiY$*g+N#;`LL_hGPQ098@%ezmMrd4G6<)lo$a!?-xekEI^Z}^yAmt0*V zU%Egorp5cyG6Fk96MkV@>t(Svk!o7qc+=>IQDa;x$8k3}Dkt$FO@fIsGk!Z{E;2li zqfxy$m8S%e7Fm<$noc+b<5`w6EBh@DCf)o(RXKOY*!lWnXOXMigwt@-;nfnBB{@1A zMK15DNk@ETq$j+=YDVt2C!?gpP4-9v!gg?(=xuSL%-obKrI(XmaQgtICd z_C9AyL_R~u5JPY6Iw2kz-3084^t}Lbv;D!zlF!+goOgpgIK4RaM1-iZ%ZSx>zMtG> z461+0F6=0=kL7ud1|;PUuO2Q$Lhtp#`xO(8r{cK#k5C`l%;$6bo*ZP)Jubrc4uLh7 zFooCW7TVaD+-2c6OS-6?e2Wz$^JEq8OWI-a6Q%~2FVFS9{=jBMY~iuwmP-X9Eaq?HiDl|LDA$v*|0w}LUd}8Vg z+n1egA(qZ#nvRy%a`yvhnB^g}#e3z;@%a6z*j%cEW_pZ!;*Pbh1%DRoid(30iob6t z3qHi#I@Uvx^OiDJx}?^M78}u~6dG=&Y+R{Znb08ZnYunBNSofE+U%Te=YH#{t0~sb zjx3h^c{lVIOE+_W_fgXsx+o3pdfZNBMs(jZqo!(EK1-p+cU7E-OyAcYNkKn8_svag>Q@j$ruM{lo2!g-}x^&0=UZ0nJ|;EfF-KEbjEZ4QUJCuldnx(Q+6pxtnZW?%$Z=`lDJLMIfO8hG!|&QRa1RVvKBdHOX!}r2jZ; zZlLaj#p7dP)k(z<5mn7-Ix1%g05KL@GKwEynpM2EaHLiv3&w&maKlB50ud=-PJqRX^>*keC9R4S z(HO^tesDND|I(R3*qt+yz1EGXLwoHq{0z^I9RZUJME66h8FrOWPhVxv^NsC94M7X;Z zX3rmf0GAiC3nyttqn*iz$;9paWzs5ji46-eaj3>4_rT;HYDdXUkr$q5=1}+tw3W5( zVELEVCGE&4F=o>&sFdltz_2REGxy)uOoXQVdsJS^5SFBM-@KRG#D!@(r&E>tT zFH>eh-c}PoH7P-7|D6OBJgY(t-Delqdexsptnq!L`;5w7pR$VRhP&AoR*E(i@R&P_ zq0AX3TkIZDjx%~Ng#rq-WRVpmPctg5zQNuvm@A0$snxN?VvrJM=2~MN3f*S&vj|=g z;}EAVQ!~A)rM=KO)TUWW8Ce?9x3ZS-GdVm~xvrB!sQ2X+|Fp|l@=*b0a%{s^>;eB&;gVXeLCBiH zTn!`sqcYRvm^DHq-!qP+_^talbh&l*kSVWGkqyQIU*wN2%1+4|A1e@i8eECx4_WSH zLYJx;(x{=nq{0W3a?rO=nd}Iu6dv=dzJpDwjGR(gVi|*P+Q8it&kx7fbKidwajAK( z>SOC@oIX!H8u40P(7@^!f3B($_Uw^GWxaLt@f(=opgvN`0e=1!k>7$Ze!qwIK1w8+ zQIapq!V&@rD`K%1*H>!Q=DsZ{U3hw2+#mkH9XgVH%^pg<9$?Drv zM>A$=iLQd&QbdwpsQC7^zDW*{7)#i3G($5c8(pG%(~%PiG$m<@)!prr7)$wo0F^KA$Pb$7%oY|vPU6{D6J`TqA>ZIM9)*qn*)x) zjlKozjwT;BuSP77*CiHf;675x77wRJ-E5Ca)o%%m4n={G)h zGQG9|27MuKc8*MhTA!OSY`UCT3#gTHv}Xcb1~Xnt|6 zTDoLg_{TNm)cWh@Sqr_&wrdC%ebmKPOz%tf^^ir~BG)E9Y@|ke@o8dhp6ZA;x&%nDb8@sPcXt96KVU%#QgZge za|@Z$W-{T+`o}+kx$V)OUn5!va*{t!Z%fqpVAXtVyXJtCJlQ8qPKuA8ms}+NP(_8p zqWq(`F9C6hJ(t8KKA>|zrQky-Q(TS1zEpL((RwqSlZApU{r)upW1*GNZLH`v^s2CY zVNyC{=pbFIRE#uadyTB29gqnCrYWFu)^oO4`~Z)KH<57f^48qnKzUb(#s<)yoG4_$ z*GHrASVhy_5&ZOrlXxOes>k-}PtUN{r#F4WJ|wYR^QQ>C(qXbOHUVOE^gWmUEikjO zpipydf2PfKY7c_*tn4RY5v#aRzF#+)?N%!sWtWx|fBOm+e*U$uy9%xK@2yN=G}T5u zjQD-!^><_jbYRDb2gSEYo1%-<&r3yM`U&TLF6~=_ydj4+|Au-Xc7R}=j}(^j^dby{%2wuS5N~hlntaw;akuhdcqH-m zbnHHwwLF`9_cl>7|9S8FxD%56(aY_x%9{CSg{OEv33vjOG9FGPT^&J%NIRzv+jXK> z=8MrtZG7IZjnu5Jj!@9vH)+AP25O)0R6aWe!AT2mt{)Kv6ow7jKJEC>x zqKRER!Cf24s!kTLkwb&&Jd>z|cZ!DHjwD>IX<6v@1$}h?2_-24MON&_7{*r*8*=wW z3m0C2u&V<4c1B2l+V!M#r{pB_2hyGN2LmzoCpIF9dC}B=jX#`*Zeg7m^%3^Dsi0>c z+$eUQIHIP@dXKe`*+2WHNo(cI*0EJh-=Rj~47}a%VtUthgOCh*G*yh0ch0SWeDdJ}{kR80&;?9sV;p~}sZzPHZ#$|p=KMypdEu_$qOSPfV#zE=G#|8ZOGi+U zL>>e8^(W7hQaFlpfb2e|rv3~HNhI_>og?{X=FWi|ie~~ii!4Z-J$qqr0V|{I8%e9W zq>cu!;o4|F!Hq?1%@=Pnez`?|`BV1J)2Bu!(;I)3)q=si9?U;}N8?dJ!%Hl+u(!{3D*RqZ|Y9+I8)9W^Z#T{IazTue!`^oIWqv)~p}cSF2&d z#?$)m39f0DVOubSj^vOmkBx4Jqs~1SR_Do2hwoLh6k*#SEd5%&#hgXt4SzD^e&1u{ zEirX%_4k#;IoJnxM@A|A)KWn)=6uk-TtbIH&ByE8fN>R$LgS-Yc;nsjmiAdFSAYDL z2`(|oy;ec?Ci~ZUETH)0a4D!rn3(S%#US08olPl0NscVDju$BxN!VkyVpBd>6HAT1 zh_&;QR~4JEL2r!A>UQ7@HA)dgwpLNpa$PX|8{PvpoKhl(ZNl%-l&8CH91X9B=mMBdpL`CZ~G?Wp0$aYbs@o%och- zr^ot%AhfUVc{uOauZoJij>A-X<2>W?t>)cs8NLV6a{4ODtjRIcP!@Gf7WZ5dN5u%m z^$`vR=PjK1=I9dq-7~W=18YBygDS6rTy}q^2|&kAEZpNc30U~z!WOy(93^N{%i;d;d+^u4KYF` zEi&W08{{4}$5vtxd(^Mq%bO_pH+;zsM5QL)fio}^{^eK72K16x5po@0FFSc0udE3K z5cqFqsAc0h>pf@(t8yb$wbIHOxP&$r;x0ZuAySf% z{L*@fV@Xn}Qo7~T7I^cX__6|*4q9T&M@i8t+y&M)#>8E5a~+SF%37BbF{EvFwVN{m>H6Fp>gP}WWmITkBjbddcK3Egn{mfiBjB8@*|gmkz@tG9Kq zC4DBh(do0Q(j^S^>tcbdn3w}Jn?yw8a**>e#IZxe5T>%<>@XJd;(d6FB-$I%c7#2( z8Bf3_*wlq5kDGh3j)BV?K7%6?{Os4SHY1M8P)wE<-> z=YeyZxz0Upac1Jbp`$Rg$cu9ifB!T$NikSo@71%_S2o~%p1M2Ft%7H{L0b5Fy-+oT zmMAuRJN(pmv!B=lc6*RHOLPIAE)#K8TDpg|6|KiAf27x?%`y>r;#(Mr-w;LNL%8H= z9MfO)Qlb~#GW)g(H5*yu7SwBU%2wgt2s~#2Zye>UReE)uUq5jKROj*+KDiA?@IJs` z+@9C9%Q*|?b_@ritlI}=8&`zLt1oT$YWgd!Qe#!4cQ}l2 z-!i(^7s;cAxlu=cgV`EG zL)19VtYd%;U#^WK-+}A;FvFw>b-7+_*-=Z5ozkDWM(mZ<+xxxNBO`kYe4E|0&gu=xr?p z6>cw5kM@in@Cj#j3my!xYq=%d4Z=39>Hr8h0woXje^Yj(B2;`cNTKgGzsnCf%d>LXu(39ArtlK#Ctd#ovvLaU&x&U$YVQn)s@ zne6sWMqGVnOle(F!ctYE5wyy{MYYcYeaWe-7i}3KOdIBb?37^<#VU`7)0Ob~VjG4s z{M~q*ku2p|uR}1UK}``lcFmBX(4FDaHZgbyDGB-z`fOg~=;UxY?8tmYOY2~3;qOiX4Qp{OIsVxD6G;r=vJ-U>R z-?^2Tmh2!W=CzDTk0FD}Z!u#kXUmf(TGpS4{0FH&Rg}28LLJf4g&|YuI*FMr8~ubF z>U#FXcFd{bB{{mW8X2xI{%Y<{tSi4Yy2i&zmQYNf0j~0@&fA(=xl=Vn)%EUq+MZ(; zi`zrzuqv*8#eOntmzG~k@Cq@9kImf5MO>jQcUYQBl;9TGTq9S4Ojr}MrN~mvX0$5u zdN7qMXEC6vEp6pnYJ!)od?X}y_ z>&4PXy#5%TN8CrB=UI`&3+_S^I?E_0R^{++umud{Dn2vGt;kPter^1>^&dv-Pg}pn zEdjv_rrFJkx>ie)H{qZC7152HdQfN zD^C_LBaTKC18G9%*;3-4I>(SPkgB6gS zR1(($A0gta5)G3Z=odTS9vsD+e^o83&~8?e185=_N;jF_BBiTtl`groFd8q9oRz2* zN9$`IzRu?hyJPU?mWySQaIM&js*@Jxg3vg`Z#7GzyDmxZaRAa$;{RYf_1xEjEp_+CUNt0-HTcp_b^(7h?BqDNw7X}xekJaA*rm`a< z9~QzAhQG2ZdVWHdnB=|Z`V3);x*%@bH~_&Hg*3r}(#$apZp-qIrR1n_UG5)~C@=8N zaTJ;sm=qN?YUVSF`JwFbZE4BC^ks=dV>c+T3azB$loGIN<&pOjyQE$9P2q}=?Vx$f zM;Vlx&}(-&>W~lU(T9}Ou_xig#qKI*4&mEV*Z1?z>&uw)i%j@>C}!E9mh(?cpM35M zx02sN|C0Y8ov*QTqGsE%7#M#_>Ab;H!?o}a_emKvNoTONHt#PnCf}ISG5TEeV6Td2 zx*!kl)|mN|OsmScj$S|PYyi&*8ow#ce~o%0_;ahCm3T@W)PIt8E!%ujFYjL`@NvoS zH%G{Xe1`E2=5N~~hddoF%l**$Et>P!HQ;HC&O=j;0U+}Bkv)vk{mp?^>4WKhSAD+U z{Gan%c#2X=D{yDi)%(cL|Jm;JcLr+y&(2_Hcc;q#xrxsQ%;4ObyGTwx)`VtL*SUGt zYhgxpD8N@py(TB_hsXPtW%In6>|oH3g5gA_UoN7yfEZhgk+{CWxBng8t}-)iC-gB5 z-AY^anz;zB(E#uQxb-U#NbLQi`o|jT)O@(-!~Mf|!`2Lx<62FgQETB%{g1V4cSh$J z>J4SX)57?muwHvx@ZO_y*SFbuq*P)8!pP#PHF9T9#h}7U_1gK;H7s|uz@yU8&!wJm ztXZ!emM*En!lD?i3nHgY0VeZXfGMl~pD%i?XRS{2fA(ALCix%m>v?MzD_w1TPp~-vLdE9`SW+X8NuJKg@YNB7ndQ7RtY=^HH~|#3O^R zp)eG_uF|7DQA+r8)LX5mVu%PFzZHB5_H^s`#S5uMqDifL^9%A@JjVY9#{TgKq5o%p zd$WCX^7{0yeH76D?qHC}|J>=fce)k*-^AygxK?HRXTHSuNVnCmoiO{GZa37Ta(t20 zE*7(y>D}&;Z$kTP$M%RiKHbLaefG%l(ZTTKVp!W}3}KH5DhlM5)28h+M%DHRqiIOY zx~<*TpEy-(hhIA3mu~o_7k=r7Uk2fqo$w2^Q#;^VxlN&On9jsAfsQ#7atxyN0vX62 zJm-WJzV((Z-+I1u4Zy(CRXWR8=`LSo`Pp~+wKGnQx8I*1?-7O3HM9JD)H)$6kBeuO zj9GMCHk(1Kb*&Xfc3pa8-f|;5jAgLx1D+vz`|@@U+XX%Op5iH;<=!~8TILR&ud*|1 z2R!@86h_Hh!Tk72E+ub*)R{t2povR%avy;5*~EJd_iD9YfBp5!^sj4kdmg^h(;+BW zJ3S}4H9oK!AIVSQ3tu0=ga$xYL`T76B1=lFBOGXWms;#alpJ#iqU~*ra}gmlVMZa= YOr;N3pXyV69?s|g10(g}%mDZT0IGmedjJ3c literal 0 HcmV?d00001 From 5a2e89a49ef3b104f55412956a7b8a74a0bbdee8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 4 Dec 2025 15:13:17 -0800 Subject: [PATCH 06/40] Customer Usage UI --- .../hooks/customers/useCustomers.ts | 41 +++++++++++++++++ .../EntityUsageExport/ExportTypeSelector.tsx | 4 +- .../EntityUsageExport/UsageExportHeader.tsx | 2 +- .../src/components/EntityUsageExport/types.ts | 3 +- .../src/components/EntityUsageExport/utils.ts | 8 ++-- .../src/components/entity_usage.test.tsx | 19 ++++++++ .../src/components/entity_usage.tsx | 19 +++++++- .../src/components/networking.tsx | 21 ++++++++- .../src/components/new_usage.test.tsx | 46 +++++++++++++++++++ .../src/components/new_usage.tsx | 22 ++++++++- 10 files changed, 173 insertions(+), 12 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts new file mode 100644 index 00000000000..10cbedc04d3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts @@ -0,0 +1,41 @@ +import { allEndUsersCall } from "@/components/networking"; +import { useQuery } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; +import { all_admin_roles } from "@/utils/roles"; + +const customersKeys = createQueryKeys("customers"); + +export interface Customer { + user_id: string; + alias?: string | null; + spend: number; + blocked: boolean; + allowed_model_region?: string | null; + default_model?: string | null; + budget_id?: string | null; + litellm_budget_table?: { + budget_id: string; + max_budget?: number | null; + soft_budget?: number | null; + max_parallel_requests?: number | null; + tpm_limit?: number | null; + rpm_limit?: number | null; + model_max_budget?: Record | null; + budget_duration?: string | null; + budget_reset_at?: string | null; + created_at: string; + created_by: string; + updated_at: string; + updated_by: string; + } | null; +} + +export type CustomersResponse = Customer[]; + +export const useCustomers = (accessToken: string | null, userRole: string | null) => { + return useQuery({ + queryKey: customersKeys.list({}), + queryFn: async () => await allEndUsersCall(accessToken!), + enabled: Boolean(accessToken) && all_admin_roles.includes(userRole || ""), + }); +}; diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.tsx index 43e6f986dfb..17a833deacb 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.tsx @@ -1,11 +1,11 @@ import React from "react"; import { Radio } from "antd"; -import type { ExportScope } from "./types"; +import type { ExportScope, EntityType } from "./types"; interface ExportTypeSelectorProps { value: ExportScope; onChange: (value: ExportScope) => void; - entityType: "tag" | "team" | "organization"; + entityType: EntityType; } const ExportTypeSelector: React.FC = ({ value, onChange, entityType }) => { diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx index 3547d65379e..e326183d880 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx @@ -7,7 +7,7 @@ import type { EntitySpendData } from "./types"; interface UsageExportHeaderProps { dateValue: DateRangePickerValue; - entityType: "tag" | "team" | "organization"; + entityType: "tag" | "team" | "organization" | "customer"; spendData: EntitySpendData; // Optional filter props showFilters?: boolean; diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/types.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/types.ts index ea11701f7ee..ded2731c945 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/types.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/types.ts @@ -2,6 +2,7 @@ import type { DateRangePickerValue } from "@tremor/react"; export type ExportFormat = "csv" | "json"; export type ExportScope = "daily" | "daily_with_models"; +export type EntityType = "tag" | "team" | "organization" | "customer"; export interface EntitySpendData { results: any[]; @@ -17,7 +18,7 @@ export interface EntitySpendData { export interface EntityUsageExportModalProps { isOpen: boolean; onClose: () => void; - entityType: "tag" | "team" | "organization"; + entityType: EntityType; spendData: EntitySpendData; dateRange: DateRangePickerValue; selectedFilters: string[]; diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts index 1327e158a6e..c93155feb23 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts @@ -1,6 +1,6 @@ import { formatNumberWithCommas } from "@/utils/dataUtils"; import Papa from "papaparse"; -import type { EntitySpendData, EntityBreakdown, ExportMetadata, ExportScope } from "./types"; +import type { EntitySpendData, EntityBreakdown, ExportMetadata, ExportScope, EntityType } from "./types"; import type { DateRangePickerValue } from "@tremor/react"; export const getEntityBreakdown = (spendData: EntitySpendData): EntityBreakdown[] => { @@ -139,7 +139,7 @@ export const generateExportData = ( }; export const generateMetadata = ( - entityType: "tag" | "team" | "organization", + entityType: EntityType, dateRange: DateRangePickerValue, selectedFilters: string[], exportScope: ExportScope, @@ -166,7 +166,7 @@ export const handleExportCSV = ( spendData: EntitySpendData, exportScope: ExportScope, entityLabel: string, - entityType: "tag" | "team" | "organization", + entityType: EntityType, ): void => { const data = generateExportData(spendData, exportScope, entityLabel); const csv = Papa.unparse(data); @@ -186,7 +186,7 @@ export const handleExportJSON = ( spendData: EntitySpendData, exportScope: ExportScope, entityLabel: string, - entityType: "tag" | "team" | "organization", + entityType: EntityType, dateRange: DateRangePickerValue, selectedFilters: string[], ): void => { diff --git a/ui/litellm-dashboard/src/components/entity_usage.test.tsx b/ui/litellm-dashboard/src/components/entity_usage.test.tsx index 17016b6479d..2c070427ead 100644 --- a/ui/litellm-dashboard/src/components/entity_usage.test.tsx +++ b/ui/litellm-dashboard/src/components/entity_usage.test.tsx @@ -18,6 +18,7 @@ vi.mock("./networking", () => ({ tagDailyActivityCall: vi.fn(), teamDailyActivityCall: vi.fn(), organizationDailyActivityCall: vi.fn(), + customerDailyActivityCall: vi.fn(), })); // Mock the child components to simplify testing @@ -42,6 +43,7 @@ describe("EntityUsage", () => { const mockTagDailyActivityCall = vi.mocked(networking.tagDailyActivityCall); const mockTeamDailyActivityCall = vi.mocked(networking.teamDailyActivityCall); const mockOrganizationDailyActivityCall = vi.mocked(networking.organizationDailyActivityCall); + const mockCustomerDailyActivityCall = vi.mocked(networking.customerDailyActivityCall); const mockSpendData = { results: [ @@ -128,9 +130,11 @@ describe("EntityUsage", () => { mockTagDailyActivityCall.mockClear(); mockTeamDailyActivityCall.mockClear(); mockOrganizationDailyActivityCall.mockClear(); + mockCustomerDailyActivityCall.mockClear(); mockTagDailyActivityCall.mockResolvedValue(mockSpendData); mockTeamDailyActivityCall.mockResolvedValue(mockSpendData); mockOrganizationDailyActivityCall.mockResolvedValue(mockSpendData); + mockCustomerDailyActivityCall.mockResolvedValue(mockSpendData); }); it("should render with tag entity type and display spend metrics", async () => { @@ -182,6 +186,21 @@ describe("EntityUsage", () => { }); }); + it("should render with customer entity type and call customer API", async () => { + render(); + + await waitFor(() => { + expect(mockCustomerDailyActivityCall).toHaveBeenCalled(); + }); + + expect(screen.getByText("Customer Spend Overview")).toBeInTheDocument(); + + await waitFor(() => { + const spendElements = screen.getAllByText("$100.50"); + expect(spendElements.length).toBeGreaterThan(0); + }); + }); + it("should switch between tabs", async () => { render(); diff --git a/ui/litellm-dashboard/src/components/entity_usage.tsx b/ui/litellm-dashboard/src/components/entity_usage.tsx index 501eac7124b..ca30ded9494 100644 --- a/ui/litellm-dashboard/src/components/entity_usage.tsx +++ b/ui/litellm-dashboard/src/components/entity_usage.tsx @@ -23,12 +23,18 @@ import { } from "@tremor/react"; import { ActivityMetrics, processActivityData } from "./activity_metrics"; import { DailyData, BreakdownMetrics, KeyMetricWithMetadata, EntityMetricWithMetadata, TagUsage } from "./usage/types"; -import { organizationDailyActivityCall, tagDailyActivityCall, teamDailyActivityCall } from "./networking"; +import { + organizationDailyActivityCall, + tagDailyActivityCall, + teamDailyActivityCall, + customerDailyActivityCall, +} from "./networking"; import TopKeyView from "./top_key_view"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { valueFormatterSpend } from "./usage/utils/value_formatters"; import { getProviderLogoAndName } from "./provider_info_helpers"; import { UsageExportHeader } from "./EntityUsageExport"; +import type { EntityType } from "./EntityUsageExport/types"; import TopModelView from "./top_model_view"; interface EntityMetrics { @@ -68,7 +74,7 @@ export interface EntityList { interface EntityUsageProps { accessToken: string | null; - entityType: "tag" | "team" | "organization"; + entityType: EntityType; entityId?: string | null; userID: string | null; userRole: string | null; @@ -135,6 +141,15 @@ const EntityUsage: React.FC = ({ selectedTags.length > 0 ? selectedTags : null, ); setSpendData(data); + } else if (entityType === "customer") { + const data = await customerDailyActivityCall( + accessToken, + startTime, + endTime, + 1, + selectedTags.length > 0 ? selectedTags : null, + ); + setSpendData(data); } else { throw new Error("Invalid entity type"); } diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 0e45c0f3a91..4582a8a8e1d 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1736,6 +1736,25 @@ export const organizationDailyActivityCall = async ( }); }; +export const customerDailyActivityCall = async ( + accessToken: string, + startTime: Date, + endTime: Date, + page: number = 1, + customerIds: string[] | null = null, +) => { + return fetchDailyActivity({ + accessToken, + endpoint: "/customer/daily/activity", + startTime, + endTime, + page, + extraQueryParams: { + end_user_ids: customerIds, + }, + }); +}; + export const getTotalSpendCall = async (accessToken: string) => { /** * Get all models on proxy @@ -2511,7 +2530,7 @@ export const allEndUsersCall = async (accessToken: string) => { console.log(data); return data; } catch (error) { - console.error("Failed to create key:", error); + console.error("Failed to fetch end users:", error); throw error; } }; diff --git a/ui/litellm-dashboard/src/components/new_usage.test.tsx b/ui/litellm-dashboard/src/components/new_usage.test.tsx index a06045137d7..aec07765e7e 100644 --- a/ui/litellm-dashboard/src/components/new_usage.test.tsx +++ b/ui/litellm-dashboard/src/components/new_usage.test.tsx @@ -3,6 +3,7 @@ import { describe, it, expect, vi, beforeEach, beforeAll } from "vitest"; import NewUsagePage from "./new_usage"; import type { Organization } from "./networking"; import * as networking from "./networking"; +import { useCustomers } from "@/app/(dashboard)/hooks/customers/useCustomers"; // Polyfill ResizeObserver for test environment beforeAll(() => { @@ -53,9 +54,14 @@ vi.mock("./EntityUsageExport", () => ({ default: () =>
Entity Usage Export Modal
, })); +vi.mock("@/app/(dashboard)/hooks/customers/useCustomers", () => ({ + useCustomers: vi.fn(), +})); + describe("NewUsage", () => { const mockUserDailyActivityAggregatedCall = vi.mocked(networking.userDailyActivityAggregatedCall); const mockTagListCall = vi.mocked(networking.tagListCall); + const mockUseCustomers = vi.mocked(useCustomers); const mockSpendData = { results: [ @@ -174,6 +180,19 @@ describe("NewUsage", () => { }, ]; + const mockCustomers = [ + { + user_id: "customer-123", + alias: "Test Customer", + spend: 0, + blocked: false, + allowed_model_region: null, + default_model: null, + budget_id: null, + litellm_budget_table: null, + }, + ]; + const defaultProps = { accessToken: "test-token", userRole: "Admin", @@ -205,6 +224,11 @@ describe("NewUsage", () => { mockTagListCall.mockClear(); mockUserDailyActivityAggregatedCall.mockResolvedValue(mockSpendData); mockTagListCall.mockResolvedValue({}); + mockUseCustomers.mockReturnValue({ + data: [], + isLoading: false, + error: null, + } as any); }); it("should render and fetch usage data on mount", async () => { @@ -289,4 +313,26 @@ describe("NewUsage", () => { expect(entityUsageElements.length).toBeGreaterThan(0); }); }); + + it("should show customer usage tab for admins", async () => { + mockUseCustomers.mockReturnValue({ + data: mockCustomers, + isLoading: false, + error: null, + } as any); + + const { getByText, getAllByText } = render(); + + await waitFor(() => { + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalled(); + }); + + const customerTab = getByText("Customer Usage"); + fireEvent.click(customerTab); + + await waitFor(() => { + const entityUsageElements = getAllByText("Entity Usage"); + expect(entityUsageElements.length).toBeGreaterThan(0); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/new_usage.tsx b/ui/litellm-dashboard/src/components/new_usage.tsx index a8d30885493..f9280ae2bf2 100644 --- a/ui/litellm-dashboard/src/components/new_usage.tsx +++ b/ui/litellm-dashboard/src/components/new_usage.tsx @@ -27,9 +27,10 @@ import { Text, Title, } from "@tremor/react"; -import React, { useCallback, useEffect, useMemo, useState } from "react"; import { Alert } from "antd"; +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { useCustomers } from "@/app/(dashboard)/hooks/customers/useCustomers"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { Button } from "@tremor/react"; import { all_admin_roles } from "../utils/roles"; @@ -86,6 +87,7 @@ const NewUsagePage: React.FC = ({ }); const [allTags, setAllTags] = useState([]); + const { data: customers = [] } = useCustomers(accessToken, userRole); const [modelViewType, setModelViewType] = useState<"groups" | "individual">("groups"); const [isCloudZeroModalOpen, setIsCloudZeroModalOpen] = useState(false); const [isGlobalExportModalOpen, setIsGlobalExportModalOpen] = useState(false); @@ -430,6 +432,7 @@ const NewUsagePage: React.FC = ({ Your Organization Usage )} Team Usage + {all_admin_roles.includes(userRole || "") ? Customer Usage : <>} {all_admin_roles.includes(userRole || "") ? Tag Usage : <>} {all_admin_roles.includes(userRole || "") ? User Agent Activity : <>} @@ -798,6 +801,23 @@ const NewUsagePage: React.FC = ({ /> + {/* Customer Usage Panel */} + + ({ + label: customer.alias || customer.user_id, + value: customer.user_id, + })) || null + } + premiumUser={premiumUser} + dateValue={dateValue} + /> + {/* Tag Usage Panel */} Date: Fri, 5 Dec 2025 12:23:25 -0800 Subject: [PATCH 07/40] change useAuthorized Hook to redirect to new login page --- .../(dashboard)/hooks/useAuthorized.test.ts | 80 +++++++++++++++++++ .../app/(dashboard)/hooks/useAuthorized.ts | 5 +- 2 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts new file mode 100644 index 00000000000..5059d5d69d1 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts @@ -0,0 +1,80 @@ +/* @vitest-environment jsdom */ +import { renderHook } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import useAuthorized from "./useAuthorized"; + +const replaceMock = vi.fn(); +const clearTokenCookiesMock = vi.fn(); +const getProxyBaseUrlMock = vi.fn(() => "http://proxy.example"); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ + replace: replaceMock, + }), +})); + +vi.mock("@/components/networking", () => ({ + getProxyBaseUrl: getProxyBaseUrlMock, +})); + +vi.mock("@/utils/cookieUtils", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + clearTokenCookies: clearTokenCookiesMock, + }; +}); + +const createJwt = (payload: Record) => { + const base64Url = btoa(JSON.stringify(payload)).replace(/=+$/, "").replace(/\+/g, "-").replace(/\//g, "_"); + return `eyJhbGciOiJub25lIn0.${base64Url}.signature`; +}; + +const clearCookie = () => { + document.cookie = "token=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;"; +}; + +describe("useAuthorized", () => { + afterEach(() => { + replaceMock.mockReset(); + clearTokenCookiesMock.mockReset(); + getProxyBaseUrlMock.mockClear(); + clearCookie(); + }); + + it("should decode the token and expose user details", () => { + const token = createJwt({ + key: "api-key-123", + user_id: "user-1", + user_email: "user@example.com", + user_role: "app_admin", + premium_user: true, + disabled_non_admin_personal_key_creation: false, + login_method: "username_password", + }); + document.cookie = `token=${token}; path=/;`; + + const { result } = renderHook(() => useAuthorized()); + + expect(result.current.token).toBe(token); + expect(result.current.accessToken).toBe("api-key-123"); + expect(result.current.userId).toBe("user-1"); + expect(result.current.userEmail).toBe("user@example.com"); + expect(result.current.userRole).toBe("Admin"); + expect(result.current.premiumUser).toBe(true); + expect(result.current.disabledPersonalKeyCreation).toBe(false); + expect(result.current.showSSOBanner).toBe(true); + expect(replaceMock).not.toHaveBeenCalled(); + }); + + it("should clear cookies and redirect on an invalid token", () => { + document.cookie = "token=invalid-token; path=/;"; + + const { result } = renderHook(() => useAuthorized()); + + expect(clearTokenCookiesMock).toHaveBeenCalled(); + expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login"); + expect(result.current.accessToken).toBeNull(); + expect(result.current.userRole).toBe("Unknown Role"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts index cba7c1a3dc8..7610c6346be 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts @@ -4,6 +4,7 @@ import { useEffect, useMemo } from "react"; import { useRouter } from "next/navigation"; import { jwtDecode } from "jwt-decode"; import { clearTokenCookies, getCookie } from "@/utils/cookieUtils"; +import { getProxyBaseUrl } from "@/components/networking"; function formatUserRole(userRole: string) { if (!userRole) { @@ -42,7 +43,7 @@ const useAuthorized = () => { // Redirect after mount if missing/invalid token useEffect(() => { if (!token) { - router.replace("/sso/key/generate"); + router.replace(`${getProxyBaseUrl()}/ui/login`); } }, [token, router]); @@ -54,7 +55,7 @@ const useAuthorized = () => { } catch { // Bad token in cookie — clear and bounce clearTokenCookies(); - router.replace("/sso/key/generate"); + router.replace(`${getProxyBaseUrl()}/ui/login`); return null; } }, [token, router]); From ac9ce4390221da0ee7ee16c55ea96f00ab499ab3 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 5 Dec 2025 12:24:22 -0800 Subject: [PATCH 08/40] Fixing test --- .../src/app/(dashboard)/hooks/useAuthorized.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts index 5059d5d69d1..9198450a63d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts @@ -3,9 +3,11 @@ import { renderHook } from "@testing-library/react"; import { afterEach, describe, expect, it, vi } from "vitest"; import useAuthorized from "./useAuthorized"; -const replaceMock = vi.fn(); -const clearTokenCookiesMock = vi.fn(); -const getProxyBaseUrlMock = vi.fn(() => "http://proxy.example"); +const { replaceMock, clearTokenCookiesMock, getProxyBaseUrlMock } = vi.hoisted(() => ({ + replaceMock: vi.fn(), + clearTokenCookiesMock: vi.fn(), + getProxyBaseUrlMock: vi.fn(() => "http://proxy.example"), +})); vi.mock("next/navigation", () => ({ useRouter: () => ({ @@ -75,6 +77,6 @@ describe("useAuthorized", () => { expect(clearTokenCookiesMock).toHaveBeenCalled(); expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login"); expect(result.current.accessToken).toBeNull(); - expect(result.current.userRole).toBe("Unknown Role"); + expect(result.current.userRole).toBe("Undefined Role"); }); }); From e21bf1982cf687ace89b8bb1db8f34fb0eb9077f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 5 Dec 2025 12:40:58 -0800 Subject: [PATCH 09/40] Fixing e2e --- .../e2e_ui_tests/view_internal_user.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts index 8be5ff0c540..832832d8ae8 100644 --- a/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts @@ -39,9 +39,8 @@ test("view internal user page", async ({ page }) => { const rowCount = await page.locator("tbody tr").count(); expect(rowCount).toBeGreaterThan(0); - const userIdHeader = page.locator("th", { hasText: "User ID" }); - page.screenshot({ path: "test-results/user_id_header.png" }); - await expect(userIdHeader).toBeVisible(); + const userIdHeader = await page.locator("th", { hasText: "User ID" }); + await expect(userIdHeader).toBeVisible({ timeout: 10000 }); // test pagination // Wait for pagination controls to be visible From a78f40f75ab59eb6f66764afdbe46e2a0fe189ff Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Dec 2025 15:25:45 -0800 Subject: [PATCH 10/40] [Fixes] Dynamic Rate Limiter - Dynamic rate limiting token count increases/decreases by 1 instead of actual count + Redis TTL (#17558) * fix async_log_success_event for _PROXY_DynamicRateLimitHandlerV3 * test_async_log_success_event_increments_by_actual_tokens * fix redis TTL * Potential fix for code scanning alert no. 3873: Clear-text logging of sensitive information Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 118 +++++++++++ .../hooks/parallel_request_limiter_v3.py | 6 + litellm/proxy/proxy_config.yaml | 65 +------ .../hooks/test_dynamic_rate_limiter_v3.py | 184 ++++++++++++++++++ 4 files changed, 315 insertions(+), 58 deletions(-) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 7e6ec1dc151..d091e348020 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -614,3 +614,121 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): f"Error in dynamic rate limiter v3 post-call hook: {str(e)}" ) return response + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + """ + Update token usage for priority-based rate limiting after successful API calls. + + Increments token counters for: + - model_saturation_check: Model-wide token tracking + - priority_model: Priority-specific token tracking + """ + from litellm.litellm_core_utils.core_helpers import ( + _get_parent_otel_span_from_kwargs, + ) + from litellm.proxy.common_utils.callback_utils import ( + get_model_group_from_litellm_kwargs, + ) + from litellm.types.caching import RedisPipelineIncrementOperation + from litellm.types.utils import Usage + + try: + verbose_proxy_logger.debug( + "INSIDE dynamic rate limiter ASYNC SUCCESS LOGGING" + ) + + litellm_parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) + + # Get metadata from standard_logging_object + standard_logging_object = kwargs.get("standard_logging_object") or {} + standard_logging_metadata = standard_logging_object.get("metadata") or {} + + # Get model and priority + model_group = get_model_group_from_litellm_kwargs(kwargs) + if not model_group: + return + + # Get priority from user_api_key_auth_metadata in standard_logging_metadata + # This is where user_api_key_dict.metadata is stored during pre-call + user_api_key_auth_metadata = standard_logging_metadata.get("user_api_key_auth_metadata") or {} + key_priority: Optional[str] = user_api_key_auth_metadata.get("priority") + + # Get total tokens from response + total_tokens = 0 + rate_limit_type = self.v3_limiter.get_rate_limit_type() + + if isinstance(response_obj, ModelResponse): + _usage = getattr(response_obj, "usage", None) + if _usage and isinstance(_usage, Usage): + if rate_limit_type == "output": + total_tokens = _usage.completion_tokens + elif rate_limit_type == "input": + total_tokens = _usage.prompt_tokens + elif rate_limit_type == "total": + total_tokens = _usage.total_tokens + + if total_tokens == 0: + return + + # Create pipeline operations for token increments + pipeline_operations: List[RedisPipelineIncrementOperation] = [] + + # Model-wide token tracking (model_saturation_check) + model_token_key = self.v3_limiter.create_rate_limit_keys( + key="model_saturation_check", + value=model_group, + rate_limit_type="tokens", + ) + pipeline_operations.append( + RedisPipelineIncrementOperation( + key=model_token_key, + increment_value=total_tokens, + ttl=self.v3_limiter.window_size, + ) + ) + + # Priority-specific token tracking (priority_model) + # Determine priority key (same logic as _get_priority_allocation) + has_explicit_priority = ( + key_priority is not None + and litellm.priority_reservation is not None + and key_priority in litellm.priority_reservation + ) + + if has_explicit_priority and key_priority is not None: + priority_key = f"{model_group}:{key_priority}" + else: + priority_key = f"{model_group}:default_pool" + + priority_token_key = self.v3_limiter.create_rate_limit_keys( + key="priority_model", + value=priority_key, + rate_limit_type="tokens", + ) + pipeline_operations.append( + RedisPipelineIncrementOperation( + key=priority_token_key, + increment_value=total_tokens, + ttl=self.v3_limiter.window_size, + ) + ) + + # Execute token increments with TTL preservation + if pipeline_operations: + await self.v3_limiter.async_increment_tokens_with_ttl_preservation( + pipeline_operations=pipeline_operations, + parent_otel_span=litellm_parent_otel_span, + ) + + # Only log 'priority' if it's known safe; otherwise, redact. + SAFE_PRIORITIES = {"low", "medium", "high", "default"} + logged_priority = key_priority if key_priority in SAFE_PRIORITIES else "REDACTED" + verbose_proxy_logger.debug( + f"[Dynamic Rate Limiter] Incremented tokens by {total_tokens} for " + f"model={model_group}, priority={logged_priority}" + ) + + except Exception as e: + verbose_proxy_logger.exception( + f"Error in dynamic rate limiter success event: {str(e)}" + ) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index c462493de6c..6ef281e5dcd 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -65,6 +65,12 @@ for i = 1, #KEYS, 2 do table.insert(results, increment_value) -- counter else local counter = redis.call('INCR', counter_key) + -- This happens when window_key exists but counter_key doesn't (e.g., tokens key + -- created after requests key when both share the same window_key) + local current_ttl = redis.call('TTL', counter_key) + if current_ttl == -1 then + redis.call('EXPIRE', counter_key, window_size) + end table.insert(results, window_start) -- window_start table.insert(results, counter) -- counter end diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 098cdb80e04..a33f56b0327 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,67 +1,16 @@ model_list: - - model_name: qwen-25vl-72b + - model_name: openai/gpt-4o-mini litellm_params: - model: bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z + model: openai/gpt-4o-mini + tpm: 1000 -guardrails: - - guardrail_name: "bedrock-pre-guard" - litellm_params: - guardrail: bedrock - mode: "pre_call" - guardrailIdentifier: ff6ujrregl1q - guardrailVersion: "DRAFT" - - -# like MCPs/vector stores -search_tools: - - search_tool_name: litellm-search - litellm_params: - search_provider: perplexity - api_key: os.environ/PERPLEXITYAI_API_KEY - - search_tool_name: firecrawl-search - litellm_params: - search_provider: firecrawl - api_key: os.environ/FIRECRAWL_API_KEY - litellm_settings: - max_end_user_budget_id: "2f6634cd-c631-4d3b-96c7-ad510ea06eaf" - # Comprehensive logging settings - store_audit_logs: true - verbose: true - log_level: "DEBUG" # Options: DEBUG, INFO, WARNING, ERROR - callbacks: ["s3_v2", "smtp_email"] - s3_callback_params: - s3_endpoint_url: "https://localhost:443" # Replace with your Minio server URL and port - s3_aws_access_key_id: "minioadmin" - s3_aws_secret_access_key: "minioadmin" - s3_region_name: "minio" # This can be any value for Minio - s3_bucket_name: "litellm-test" # Replace with your bucket name - s3_use_ssl: False - s3_verify: False - cache: True - cache_params: - type: local - drop_params: True + callbacks: ["dynamic_rate_limiter_v3"] + priority_reservation: + "prod": 0.9 # 90% reserved for production + "dev": 0.1 # 10% reserved for development -general_settings: - store_prompts_in_spend_logs: True - pass_through_endpoints: - - path: "/special/rerank" - target: "https://api.cohere.com/v1/rerank" - headers: - Authorization: "Bearer os.environ/COHERE_API_KEY" - guardrails: - bedrock-pre-guard: - request_fields: ["documents[*].text"] -vector_store_registry: - - vector_store_name: "bedrock-litellm-website-knowledgebase" - litellm_params: - vector_store_id: "T37J8R4WTM" - custom_llm_provider: "bedrock" - vector_store_description: "Bedrock vector store for the Litellm website knowledgebase" - vector_store_metadata: - source: "https://www.litellm.com/docs" diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 1f76013e237..d9e10e6f4b8 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1323,3 +1323,187 @@ async def test_default_priority_shared_pool(): print(f" - 3 keys without priority share ONE pool: {desc_a[0]['value']}") print(f" - Shared pool limit: {desc_a[0]['rate_limit']['requests_per_unit']} RPM") print(f" - Explicit priority 'prod' uses separate pool: {desc_prod[0]['value']}") + + +@pytest.mark.asyncio +async def test_async_log_success_event_increments_by_actual_tokens(): + """ + Test that async_log_success_event increments token counters by actual token usage. + + This validates the fix for Bug 1: Token count was incrementing by 1 instead of actual usage. + The async_log_success_event should increment both model_saturation_check and priority_model + counters by the actual completion_tokens (when rate_limit_type=output). + """ + from unittest.mock import MagicMock + + from litellm.types.utils import ModelResponse, Usage + + os.environ["LITELLM_LICENSE"] = "test-license-key" + litellm.priority_reservation = {"dev": 0.1, "prod": 0.9} + + dual_cache = DualCache() + handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache) + + model = "test-token-increment" + llm_router = Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "test-key", + "api_base": "test-base", + "tpm": 1000, + }, + } + ] + ) + handler.update_variables(llm_router=llm_router) + + # Track what gets incremented + increment_calls = [] + + async def mock_increment(pipeline_operations, parent_otel_span=None): + for op in pipeline_operations: + increment_calls.append({ + "key": op["key"], + "increment_value": op["increment_value"], + }) + + handler.v3_limiter.async_increment_tokens_with_ttl_preservation = mock_increment + + # Create mock response with 50 completion tokens + mock_response = MagicMock(spec=ModelResponse) + mock_response.usage = MagicMock(spec=Usage) + mock_response.usage.prompt_tokens = 10 + mock_response.usage.completion_tokens = 50 + mock_response.usage.total_tokens = 60 + + # Create kwargs with priority in user_api_key_auth_metadata + kwargs = { + "standard_logging_object": { + "metadata": { + "user_api_key_auth_metadata": {"priority": "dev"}, + }, + "model_group": model, + }, + "litellm_params": { + "metadata": {"model_group": model}, + }, + } + + with patch( + "litellm.proxy.common_utils.callback_utils.get_model_group_from_litellm_kwargs", + return_value=model, + ): + await handler.async_log_success_event( + kwargs=kwargs, + response_obj=mock_response, + start_time=None, + end_time=None, + ) + + # Verify increments happened with actual token count (50 completion tokens) + assert len(increment_calls) == 2, f"Expected 2 increment calls, got {len(increment_calls)}" + + # Both should increment by 50 (completion_tokens, since rate_limit_type defaults to 'output') + for call in increment_calls: + assert call["increment_value"] == 50, ( + f"Expected increment of 50 tokens, got {call['increment_value']} for key {call['key']}" + ) + + # Verify correct keys were used + keys = [call["key"] for call in increment_calls] + assert any("model_saturation_check" in k for k in keys), "Should increment model_saturation_check" + assert any("priority_model" in k and "dev" in k for k in keys), "Should increment priority_model with 'dev' priority" + + +@pytest.mark.asyncio +async def test_async_log_success_event_uses_team_priority_from_auth_metadata(): + """ + Test that async_log_success_event correctly retrieves priority from user_api_key_auth_metadata. + + This validates the fix where priority is retrieved from standard_logging_metadata.user_api_key_auth_metadata + instead of just standard_logging_metadata.priority. This is important for team-based priority inheritance. + """ + from unittest.mock import MagicMock + + from litellm.types.utils import ModelResponse, Usage + + os.environ["LITELLM_LICENSE"] = "test-license-key" + litellm.priority_reservation = {"team_priority": 0.8, "default": 0.2} + + dual_cache = DualCache() + handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache) + + model = "test-team-priority" + llm_router = Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "test-key", + "api_base": "test-base", + "tpm": 1000, + }, + } + ] + ) + handler.update_variables(llm_router=llm_router) + + # Track incremented keys to verify priority is used correctly + incremented_keys = [] + + async def mock_increment(pipeline_operations, parent_otel_span=None): + for op in pipeline_operations: + incremented_keys.append(op["key"]) + + handler.v3_limiter.async_increment_tokens_with_ttl_preservation = mock_increment + + # Create mock response + mock_response = MagicMock(spec=ModelResponse) + mock_response.usage = MagicMock(spec=Usage) + mock_response.usage.prompt_tokens = 10 + mock_response.usage.completion_tokens = 20 + mock_response.usage.total_tokens = 30 + + # Simulate team metadata inheritance: priority is in user_api_key_auth_metadata + # This is how the proxy passes team metadata to the callback + kwargs = { + "standard_logging_object": { + "metadata": { + # Priority NOT at top level (this would fail before the fix) + # Priority IS in user_api_key_auth_metadata (team inheritance) + "user_api_key_auth_metadata": {"priority": "team_priority"}, + }, + "model_group": model, + }, + "litellm_params": { + "metadata": {"model_group": model}, + }, + } + + with patch( + "litellm.proxy.common_utils.callback_utils.get_model_group_from_litellm_kwargs", + return_value=model, + ): + await handler.async_log_success_event( + kwargs=kwargs, + response_obj=mock_response, + start_time=None, + end_time=None, + ) + + # Verify the priority_model key uses 'team_priority' (not 'default_pool') + priority_keys = [k for k in incremented_keys if "priority_model" in k] + assert len(priority_keys) == 1, f"Expected 1 priority_model key, got {len(priority_keys)}" + + # The key should contain 'team_priority', not 'default_pool' + assert "team_priority" in priority_keys[0], ( + f"Expected priority key to use 'team_priority' from user_api_key_auth_metadata, " + f"got key: {priority_keys[0]}" + ) + assert "default_pool" not in priority_keys[0], ( + f"Priority key should NOT use 'default_pool', should use team's priority. Got: {priority_keys[0]}" + ) From 769f3cc310a71be0b19f8b9c5aeb611578ee52f9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Dec 2025 15:26:00 -0800 Subject: [PATCH 11/40] [Bug fix] Secret Managers Integration - Make email and secret manager operations independent in key management hooks (#17551) * TestKeyManagementEventHooksIndependentOperations * KeyManagementEventHooks - make ops independant --- .../proxy/hooks/key_management_event_hooks.py | 216 ++++++++++++------ .../hooks/test_key_management_event_hooks.py | 130 +++++++++++ 2 files changed, 276 insertions(+), 70 deletions(-) create mode 100644 tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index 44be6bbe656..5cfc85ae7aa 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -45,9 +45,13 @@ class KeyManagementEventHooks: ) from litellm.proxy.proxy_server import litellm_proxy_admin_name - await KeyManagementEventHooks._send_key_created_email( - response.model_dump(exclude_none=True) - ) + # Send email notification - non-blocking, independent operation + try: + await KeyManagementEventHooks._send_key_created_email( + response.model_dump(exclude_none=True) + ) + except Exception as e: + verbose_proxy_logger.warning(f"Failed to send key created email: {e}") # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: @@ -69,11 +73,17 @@ class KeyManagementEventHooks: ) ) ) - # store the generated key in the secret manager - await KeyManagementEventHooks._store_virtual_key_in_secret_manager( - secret_name=data.key_alias or f"virtual-key-{response.token_id}", - secret_token=response.key, - ) + + # Store the generated key in the secret manager - non-blocking, independent operation + try: + await KeyManagementEventHooks._store_virtual_key_in_secret_manager( + secret_name=data.key_alias or f"virtual-key-{response.token_id}", + secret_token=response.key, + ) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to store virtual key in secret manager: {e}" + ) @staticmethod async def async_key_updated_hook( @@ -132,22 +142,31 @@ class KeyManagementEventHooks: ) from litellm.proxy.proxy_server import litellm_proxy_admin_name - # store the generated key in the secret manager + # Store the generated key in the secret manager - non-blocking, independent operation if data is not None and response.token_id is not None: - initial_secret_name = ( - existing_key_row.key_alias or f"virtual-key-{existing_key_row.token}" - ) - await KeyManagementEventHooks._rotate_virtual_key_in_secret_manager( - current_secret_name=initial_secret_name, - new_secret_name=data.key_alias or f"virtual-key-{response.token_id}", - new_secret_value=response.key, - ) + try: + initial_secret_name = ( + existing_key_row.key_alias + or f"virtual-key-{existing_key_row.token}" + ) + await KeyManagementEventHooks._rotate_virtual_key_in_secret_manager( + current_secret_name=initial_secret_name, + new_secret_name=data.key_alias or f"virtual-key-{response.token_id}", + new_secret_value=response.key, + ) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to rotate virtual key in secret manager: {e}" + ) - # send key rotated email if configured - await KeyManagementEventHooks._send_key_rotated_email( - response=response.model_dump(exclude_none=True), - existing_key_alias=existing_key_row.key_alias, - ) + # Send key rotated email if configured - non-blocking, independent operation + try: + await KeyManagementEventHooks._send_key_rotated_email( + response=response.model_dump(exclude_none=True), + existing_key_alias=existing_key_row.key_alias, + ) + except Exception as e: + verbose_proxy_logger.warning(f"Failed to send key rotated email: {e}") # store the audit log if litellm.store_audit_logs is True and existing_key_row.token is not None: @@ -324,66 +343,109 @@ class KeyManagementEventHooks: ) @staticmethod - async def _send_key_created_email(response: dict): + def _is_email_sending_enabled() -> bool: + """ + Check if email sending is enabled via v2 enterprise loggers or v0 alerting config. + + Returns True only if email is actually configured, preventing any email + processing when the user has not opted in. + """ + # Check v2 enterprise email loggers try: from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( BaseEmailLogger, ) - except ImportError: - raise Exception( - "Trying to use Email Hooks" - + CommonProxyErrors.missing_enterprise_package.value + + initialized_email_loggers = ( + litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=BaseEmailLogger + ) ) + if len(initialized_email_loggers) > 0: + return True + except ImportError: + pass + + # Check v0 alerting config + from litellm.proxy.proxy_server import general_settings + + if "email" in general_settings.get("alerting", []): + return True + + return False + + @staticmethod + async def _send_key_created_email(response: dict): + """ + Send key created email if email sending is enabled. + + This method is non-blocking - it will return silently if email is not + configured, and will log warnings instead of raising exceptions on failure. + """ + # Early exit if email is not enabled + if not KeyManagementEventHooks._is_email_sending_enabled(): + verbose_proxy_logger.debug( + "Email sending not enabled, skipping key created email" + ) + return from litellm.proxy.proxy_server import general_settings, proxy_logging_obj + ########################## + # v2 integration for emails (enterprise) + ########################## try: + from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( + BaseEmailLogger, + ) from litellm_enterprise.types.enterprise_callbacks.send_emails import ( SendKeyCreatedEmailEvent, ) + + initialized_email_loggers = ( + litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=BaseEmailLogger + ) + ) + if len(initialized_email_loggers) > 0: + event = SendKeyCreatedEmailEvent( + virtual_key=response.get("key", ""), + event="key_created", + event_group=Litellm_EntityType.KEY, + event_message="API Key Created", + token=response.get("token", ""), + spend=response.get("spend", 0.0), + max_budget=response.get("max_budget", 0.0), + user_id=response.get("user_id", None), + team_id=response.get("team_id", "Default Team"), + key_alias=response.get("key_alias", None), + ) + for email_logger in initialized_email_loggers: + if isinstance(email_logger, BaseEmailLogger): + await email_logger.send_key_created_email( + send_key_created_email_event=event, + ) + return except ImportError: - raise Exception( - "Trying to use Email Hooks" - + CommonProxyErrors.missing_enterprise_package.value - ) - - event = SendKeyCreatedEmailEvent( - virtual_key=response.get("key", ""), - event="key_created", - event_group=Litellm_EntityType.KEY, - event_message="API Key Created", - token=response.get("token", ""), - spend=response.get("spend", 0.0), - max_budget=response.get("max_budget", 0.0), - user_id=response.get("user_id", None), - team_id=response.get("team_id", "Default Team"), - key_alias=response.get("key_alias", None), - ) - - ########################## - # v2 integration for emails - ########################## - initialized_email_loggers = ( - litellm.logging_callback_manager.get_custom_loggers_for_type( - callback_type=BaseEmailLogger - ) - ) - if len(initialized_email_loggers) > 0: - for email_logger in initialized_email_loggers: - if isinstance(email_logger, BaseEmailLogger): - await email_logger.send_key_created_email( - send_key_created_email_event=event, - ) + pass ########################## # v0 integration for emails ########################## - else: - if "email" not in general_settings.get("alerting", []): - raise ValueError( - "Email alerting not setup on config.yaml. Please set `alerting=['email']. \nDocs: https://docs.litellm.ai/docs/proxy/email`" - ) + if "email" in general_settings.get("alerting", []): + from litellm.proxy._types import WebhookEvent + event = WebhookEvent( + event="key_created", + event_group=Litellm_EntityType.KEY, + event_message="API Key Created", + token=response.get("token", ""), + spend=response.get("spend", 0.0), + max_budget=response.get("max_budget", 0.0), + user_id=response.get("user_id", None), + team_id=response.get("team_id", "Default Team"), + key_alias=response.get("key_alias", None), + ) # If user configured email alerting - send an Email letting their end-user know the key was created asyncio.create_task( proxy_logging_obj.slack_alerting_instance.send_key_created_or_user_invited_email( @@ -393,25 +455,39 @@ class KeyManagementEventHooks: @staticmethod async def _send_key_rotated_email(response: dict, existing_key_alias: Optional[str]): + """ + Send key rotated email if email sending is enabled. + + This method is non-blocking - it will return silently if email is not + configured, and will log warnings instead of raising exceptions on failure. + """ + # Early exit if email is not enabled + if not KeyManagementEventHooks._is_email_sending_enabled(): + verbose_proxy_logger.debug( + "Email sending not enabled, skipping key rotated email" + ) + return + try: from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( BaseEmailLogger, ) except ImportError: - raise Exception( - "Trying to use Email Hooks" - + CommonProxyErrors.missing_enterprise_package.value + # Enterprise package not installed - v0 doesn't support key rotated email + verbose_proxy_logger.debug( + "Enterprise package not installed, skipping key rotated email" ) + return try: from litellm_enterprise.types.enterprise_callbacks.send_emails import ( SendKeyRotatedEmailEvent, ) except ImportError: - raise Exception( - "Trying to use Email Hooks" - + CommonProxyErrors.missing_enterprise_package.value + verbose_proxy_logger.debug( + "Enterprise types not available, skipping key rotated email" ) + return event = SendKeyRotatedEmailEvent( virtual_key=response.get("key", ""), diff --git a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py new file mode 100644 index 00000000000..f731d9e298a --- /dev/null +++ b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py @@ -0,0 +1,130 @@ +""" +Tests for KeyManagementEventHooks. + +Validates that email and secret manager operations are independent and non-blocking. +""" + +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks + + +class TestKeyManagementEventHooksIndependentOperations: + """Tests that email and secret manager operations are independent.""" + + @pytest.mark.asyncio + async def test_email_failure_does_not_block_secret_manager(self): + """ + Test that if email sending fails, secret manager operation still runs. + + This validates the independent operation design where one failure + does not block the other operation. + """ + secret_manager_called = {"called": False} + + # Mock the email method to raise an exception + async def mock_send_email_raises(*args, **kwargs): + raise Exception("Email service unavailable") + + # Mock the secret manager method to track if it was called + async def mock_store_secret(*args, **kwargs): + secret_manager_called["called"] = True + + # Create mock objects for the hook parameters + mock_data = MagicMock() + mock_data.key_alias = "test-key-alias" + + mock_response = MagicMock() + mock_response.model_dump.return_value = {"key": "sk-test", "token": "test-token"} + mock_response.model_dump_json.return_value = '{"key": "sk-test"}' + mock_response.token_id = "token-123" + mock_response.key = "sk-test-key" + + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "user-123" + mock_user_api_key_dict.api_key = "api-key-123" + + with patch.object( + KeyManagementEventHooks, + "_send_key_created_email", + side_effect=mock_send_email_raises, + ), patch.object( + KeyManagementEventHooks, + "_store_virtual_key_in_secret_manager", + side_effect=mock_store_secret, + ), patch( + "litellm.store_audit_logs", False + ), patch( + "litellm.proxy.hooks.key_management_event_hooks.verbose_proxy_logger" + ): + # Should not raise even though email fails + await KeyManagementEventHooks.async_key_generated_hook( + data=mock_data, + response=mock_response, + user_api_key_dict=mock_user_api_key_dict, + ) + + # Secret manager should have been called despite email failure + assert secret_manager_called["called"] is True + + @pytest.mark.asyncio + async def test_secret_manager_failure_does_not_block_email(self): + """ + Test that if secret manager fails, email operation still runs. + + This validates the independent operation design where one failure + does not block the other operation. + """ + email_called = {"called": False} + + # Mock the email method to track if it was called + async def mock_send_email(*args, **kwargs): + email_called["called"] = True + + # Mock the secret manager method to raise an exception + async def mock_store_secret_raises(*args, **kwargs): + raise Exception("Secret manager unavailable") + + # Create mock objects for the hook parameters + mock_data = MagicMock() + mock_data.key_alias = "test-key-alias" + + mock_response = MagicMock() + mock_response.model_dump.return_value = {"key": "sk-test", "token": "test-token"} + mock_response.model_dump_json.return_value = '{"key": "sk-test"}' + mock_response.token_id = "token-123" + mock_response.key = "sk-test-key" + + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "user-123" + mock_user_api_key_dict.api_key = "api-key-123" + + with patch.object( + KeyManagementEventHooks, + "_send_key_created_email", + side_effect=mock_send_email, + ), patch.object( + KeyManagementEventHooks, + "_store_virtual_key_in_secret_manager", + side_effect=mock_store_secret_raises, + ), patch( + "litellm.store_audit_logs", False + ), patch( + "litellm.proxy.hooks.key_management_event_hooks.verbose_proxy_logger" + ): + # Should not raise even though secret manager fails + await KeyManagementEventHooks.async_key_generated_hook( + data=mock_data, + response=mock_response, + user_api_key_dict=mock_user_api_key_dict, + ) + + # Email should have been called despite secret manager failure + assert email_called["called"] is True + From 7259de2f12360ef8040b95474368dd67f49d3855 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Fri, 5 Dec 2025 20:26:20 -0300 Subject: [PATCH 12/40] feat: add Mistral Large 3 model support (#17547) Add Mistral Large 3 (675B MoE) to model catalog for both providers: - mistral/mistral-large-3 - azure_ai/mistral-large-3 Specs: - 256k context window - $0.50/1M input, $1.50/1M output - Supports vision (multimodal) - Supports function calling Closes #17527 --- model_prices_and_context_window.json | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 634ea6dc48a..d4afde20e93 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5164,6 +5164,19 @@ "supports_function_calling": true, "supports_tool_choice": true }, + "azure_ai/mistral-large-3": { + "input_cost_per_token": 5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 256000, + "max_output_tokens": 8191, + "max_tokens": 8191, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://azure.microsoft.com/en-us/blog/introducing-mistral-large-3-in-microsoft-foundry-open-capable-and-ready-for-production-workloads/", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure_ai/mistral-medium-2505": { "input_cost_per_token": 4e-07, "litellm_provider": "azure_ai", @@ -18745,6 +18758,21 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/mistral-large-3": { + "input_cost_per_token": 5e-07, + "litellm_provider": "mistral", + "max_input_tokens": 256000, + "max_output_tokens": 8191, + "max_tokens": 8191, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "mistral/mistral-medium": { "input_cost_per_token": 2.7e-06, "litellm_provider": "mistral", From 6ff7ed14f693d1360546891ede68b519eea003fc Mon Sep 17 00:00:00 2001 From: Devaj Mody Date: Fri, 5 Dec 2025 18:30:59 -0500 Subject: [PATCH 13/40] fix(team): use organization.members instead of deprecated organization.users (#17557) Fixes #17552 - Change Prisma include from 'users' to 'members' - Use LiteLLM_OrganizationTableWithMembers type for membership validation - Access organization.members instead of organization.users - Add tests for membership validation --- .../management_endpoints/team_endpoints.py | 9 +- .../test_team_endpoints.py | 114 +++++++++++++++++- 2 files changed, 117 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 4b62e490a82..9009ce8995b 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -31,6 +31,7 @@ from litellm.proxy._types import ( LiteLLM_ManagementEndpoint_MetadataFields_Premium, LiteLLM_ModelTable, LiteLLM_OrganizationTable, + LiteLLM_OrganizationTableWithMembers, LiteLLM_TeamMembership, LiteLLM_TeamTable, LiteLLM_TeamTableCachedObj, @@ -1051,7 +1052,7 @@ async def fetch_and_validate_organization( organization_row = await prisma_client.db.litellm_organizationtable.find_unique( where={"organization_id": organization_id}, - include={"litellm_budget_table": True, "users": True}, + include={"litellm_budget_table": True, "members": True}, ) if organization_row is None: @@ -1064,7 +1065,7 @@ async def fetch_and_validate_organization( validate_team_org_change( team=LiteLLM_TeamTable(**existing_team_row.model_dump()), - organization=LiteLLM_OrganizationTable(**organization_row.model_dump()), + organization=LiteLLM_OrganizationTableWithMembers(**organization_row.model_dump()), llm_router=llm_router, ) @@ -1072,7 +1073,7 @@ async def fetch_and_validate_organization( def validate_team_org_change( - team: LiteLLM_TeamTable, organization: LiteLLM_OrganizationTable, llm_router: Router + team: LiteLLM_TeamTable, organization: LiteLLM_OrganizationTableWithMembers, llm_router: Router ) -> bool: """ Validate that a team can be moved to an organization. @@ -1123,7 +1124,7 @@ def validate_team_org_change( # Check if the team's user_id is a member of the org team_members = [m.user_id for m in team.members_with_roles] - org_members = [m.user_id for m in organization.users] if organization.users else [] + org_members = [m.user_id for m in organization.members] if organization.members else [] not_in_org = [ m for m in team_members diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 06ec71a84f8..d096b5515a0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -16,7 +16,9 @@ sys.path.insert( ) # Adds the parent directory to the system path from litellm.proxy._types import UserAPIKeyAuth # Import UserAPIKeyAuth from litellm.proxy._types import ( + LiteLLM_OrganizationMembershipTable, LiteLLM_OrganizationTable, + LiteLLM_OrganizationTableWithMembers, LiteLLM_TeamTable, LitellmUserRoles, Member, @@ -95,7 +97,7 @@ async def test_validate_team_org_change_same_org_id(): team.members_with_roles = [] # Mock organization - organization = MagicMock(spec=LiteLLM_OrganizationTable) + organization = MagicMock(spec=LiteLLM_OrganizationTableWithMembers) organization.organization_id = org_id organization.models = [] organization.litellm_budget_table = MagicMock() @@ -108,7 +110,7 @@ async def test_validate_team_org_change_same_org_id(): organization.litellm_budget_table.rpm_limit = ( 50 # This would normally fail validation ) - organization.users = [] + organization.members = [] # Mock Router mock_router = MagicMock(spec=Router) @@ -126,6 +128,114 @@ async def test_validate_team_org_change_same_org_id(): mock_access_check.assert_not_called() # Ensure access check wasn't called +@pytest.mark.asyncio +async def test_validate_team_org_change_members_in_org(): + """ + Test that validate_team_org_change passes when team members are in organization.members. + + This tests the fix for issue #17552 where membership was incorrectly checked against + organization.users (deprecated) instead of organization.members (correct). + """ + team_org_id = "team-org-123" + new_org_id = "new-org-456" + user_id_1 = "user-123" + user_id_2 = "user-456" + + # Mock team with members + team = MagicMock(spec=LiteLLM_TeamTable) + team.organization_id = team_org_id + team.models = [] + team.max_budget = None + team.tpm_limit = None + team.rpm_limit = None + + # Create mock team members + team_member_1 = MagicMock() + team_member_1.user_id = user_id_1 + team_member_2 = MagicMock() + team_member_2.user_id = user_id_2 + team.members_with_roles = [team_member_1, team_member_2] + + # Mock organization with members (using LiteLLM_OrganizationMembershipTable structure) + organization = MagicMock(spec=LiteLLM_OrganizationTableWithMembers) + organization.organization_id = new_org_id + organization.models = [] + organization.litellm_budget_table = None + + # Create mock organization members - these should match team members + org_member_1 = MagicMock(spec=LiteLLM_OrganizationMembershipTable) + org_member_1.user_id = user_id_1 + org_member_2 = MagicMock(spec=LiteLLM_OrganizationMembershipTable) + org_member_2.user_id = user_id_2 + organization.members = [org_member_1, org_member_2] + + # Mock Router + mock_router = MagicMock(spec=Router) + + # Test should pass - all team members are in org members + result = validate_team_org_change( + team=team, organization=organization, llm_router=mock_router + ) + assert result is True + + +@pytest.mark.asyncio +async def test_validate_team_org_change_member_not_in_org(): + """ + Test that validate_team_org_change raises HTTPException when team members + are NOT in organization.members. + + This tests the fix for issue #17552 where membership was incorrectly checked against + organization.users (deprecated) instead of organization.members (correct). + """ + team_org_id = "team-org-123" + new_org_id = "new-org-456" + user_id_1 = "user-123" + user_id_2 = "user-456" + user_id_not_in_org = "user-not-in-org-789" + + # Mock team with members (including one not in org) + team = MagicMock(spec=LiteLLM_TeamTable) + team.organization_id = team_org_id + team.models = [] + team.max_budget = None + team.tpm_limit = None + team.rpm_limit = None + + # Create mock team members - user_id_not_in_org is not in the org + team_member_1 = MagicMock() + team_member_1.user_id = user_id_1 + team_member_2 = MagicMock() + team_member_2.user_id = user_id_not_in_org + team.members_with_roles = [team_member_1, team_member_2] + + # Mock organization with members (missing user_id_not_in_org) + organization = MagicMock(spec=LiteLLM_OrganizationTableWithMembers) + organization.organization_id = new_org_id + organization.models = [] + organization.litellm_budget_table = None + + # Create mock organization members - only user_id_1 and user_id_2 are members + org_member_1 = MagicMock(spec=LiteLLM_OrganizationMembershipTable) + org_member_1.user_id = user_id_1 + org_member_2 = MagicMock(spec=LiteLLM_OrganizationMembershipTable) + org_member_2.user_id = user_id_2 + organization.members = [org_member_1, org_member_2] + + # Mock Router + mock_router = MagicMock(spec=Router) + + # Test should fail - user_id_not_in_org is not in org members + with pytest.raises(HTTPException) as exc_info: + validate_team_org_change( + team=team, organization=organization, llm_router=mock_router + ) + + assert exc_info.value.status_code == 403 + assert "not a member of the organization" in str(exc_info.value.detail) + assert user_id_not_in_org in str(exc_info.value.detail) + + # Test for /team/permissions_list endpoint (GET) @pytest.mark.asyncio async def test_get_team_permissions_list_success(mock_db_client, mock_admin_auth): From f02df3035a331ca214080a08afd83a250d596491 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Dec 2025 15:42:27 -0800 Subject: [PATCH 14/40] [Feat] Allow using dynamic rate limit/priority reservation on teams (#17061) * use helper to get key/team priority * test_team_metadata_priority * docs team priority --- .../docs/proxy/dynamic_rate_limit.md | 39 ++++++++++++- .../proxy/hooks/dynamic_rate_limiter_v3.py | 56 ++++++++++++++----- 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/docs/my-website/docs/proxy/dynamic_rate_limit.md b/docs/my-website/docs/proxy/dynamic_rate_limit.md index 9c875a51eba..f5438b5a6f5 100644 --- a/docs/my-website/docs/proxy/dynamic_rate_limit.md +++ b/docs/my-website/docs/proxy/dynamic_rate_limit.md @@ -175,7 +175,37 @@ general_settings: litellm --config /path/to/config.yaml ``` -#### 2. Create Keys with Priority Levels +### Set priority on either a team or a key + +Priority can be set at either the **team level** or **key level**. Team-level priority takes precedence over key-level priority. + +**Option A: Set Priority on Team (Recommended)** + +All keys within a team will inherit the team's priority. This is useful when you want all keys for a specific environment or project to have the same priority. + +```bash +curl -X POST 'http://0.0.0.0:4000/team/new' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "team_alias": "production-team", + "metadata": {"priority": "prod"} +}' +``` + +Create a key for this team: +```bash +curl -X POST 'http://0.0.0.0:4000/key/generate' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "team_id": "team-id-from-previous-response" +}' +``` + +**Option B: Set Priority on Individual Keys** + +Set priority directly on the key. This is useful when you need fine-grained control per key. **Production Key:** ```bash @@ -205,7 +235,7 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \ -d '{}' ``` -**Expected Response for both:** +**Expected Response:** ```json { "key": "sk-...", @@ -214,6 +244,11 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \ } ``` +**Priority Resolution Order:** +1. If key belongs to a team with `metadata.priority` set → use team priority +2. Else if key has `metadata.priority` set → use key priority +3. Else → use `default_priority` from config + #### 3. Test Priority Allocation **Test Production Key (should get 9 RPM):** diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index d091e348020..53419ef6ad7 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -80,6 +80,32 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): weight = convert_priority_to_percent(value, model_info) return weight + def _get_priority_from_user_api_key_dict( + self, user_api_key_dict: UserAPIKeyAuth + ) -> Optional[str]: + """ + Get priority from user_api_key_dict. + + Checks team metadata first (takes precedence), then falls back to key metadata. + + Args: + user_api_key_dict: User authentication info + + Returns: + Priority string if found, None otherwise + """ + priority: Optional[str] = None + + # Check team metadata first (takes precedence) + if user_api_key_dict.team_metadata is not None: + priority = user_api_key_dict.team_metadata.get("priority", None) + + # Fall back to key metadata + if priority is None: + priority = user_api_key_dict.metadata.get("priority", None) + + return priority + def _normalize_priority_weights( self, model_info: ModelGroupInfo ) -> Dict[str, float]: @@ -328,7 +354,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): model: str, model_group_info: ModelGroupInfo, user_api_key_dict: UserAPIKeyAuth, - key_priority: Optional[str], + priority: Optional[str], saturation: float, data: dict, ) -> None: @@ -355,7 +381,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): model: Model name model_group_info: Model configuration user_api_key_dict: User authentication info - key_priority: User's priority level + priority: User's priority level saturation: Current saturation level data: Request data dictionary @@ -384,7 +410,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): priority_descriptors = self._create_priority_based_descriptors( model=model, user_api_key_dict=user_api_key_dict, - priority=key_priority, + priority=priority, ) if priority_descriptors: descriptors_to_check.extend(priority_descriptors) @@ -412,14 +438,14 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): status_code=429, detail={ "error": f"Model capacity reached for {model}. " - f"Priority: {key_priority}, " + f"Priority: {priority}, " f"Rate limit type: {status['rate_limit_type']}, " f"Remaining: {status['limit_remaining']}" }, headers={ "retry-after": str(self.v3_limiter.window_size), "rate_limit_type": str(status["rate_limit_type"]), - "x-litellm-priority": key_priority or "default", + "x-litellm-priority": priority or "default", }, ) @@ -427,13 +453,13 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): elif descriptor_key == "priority_model" and should_enforce_priority: verbose_proxy_logger.debug( f"Enforcing priority limits for {model}, saturation: {saturation:.1%}, " - f"priority: {key_priority}" + f"priority: {priority}" ) raise HTTPException( status_code=429, detail={ "error": f"Priority-based rate limit exceeded. " - f"Priority: {key_priority}, " + f"Priority: {priority}, " f"Rate limit type: {status['rate_limit_type']}, " f"Remaining: {status['limit_remaining']}, " f"Model saturation: {saturation:.1%}" @@ -441,7 +467,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): headers={ "retry-after": str(self.v3_limiter.window_size), "rate_limit_type": str(status["rate_limit_type"]), - "x-litellm-priority": key_priority or "default", + "x-litellm-priority": priority or "default", "x-litellm-saturation": f"{saturation:.2%}", }, ) @@ -521,7 +547,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return None model = data["model"] - key_priority: Optional[str] = user_api_key_dict.metadata.get("priority", None) + priority = self._get_priority_from_user_api_key_dict( + user_api_key_dict=user_api_key_dict + ) # Get model configuration model_group_info: Optional[ModelGroupInfo] = ( @@ -543,7 +571,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): verbose_proxy_logger.debug( f"[Dynamic Rate Limiter] Model={model}, Saturation={saturation:.1%}, " - f"Threshold={saturation_threshold:.1%}, Priority={key_priority}" + f"Threshold={saturation_threshold:.1%}, Priority={priority}" ) # STEP 2: Check rate limits in THREE phases @@ -555,7 +583,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): model=model, model_group_info=model_group_info, user_api_key_dict=user_api_key_dict, - key_priority=key_priority, + priority=priority, saturation=saturation, data=data, ) @@ -586,8 +614,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # Add additional priority-specific headers if isinstance(response, ModelResponse): - key_priority: Optional[str] = user_api_key_dict.metadata.get( - "priority", None + priority = self._get_priority_from_user_api_key_dict( + user_api_key_dict=user_api_key_dict ) # Get existing additional headers @@ -599,7 +627,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ) # Add priority information - additional_headers["x-litellm-priority"] = key_priority or "default" + additional_headers["x-litellm-priority"] = priority or "default" additional_headers["x-litellm-rate-limiter-version"] = "v3" # Update response From 5fb7530d8c28c040ef10ec1a53f7f67d6efb266c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:44:15 -0800 Subject: [PATCH 15/40] build(deps): bump jws from 3.2.2 to 3.2.3 in /ui/litellm-dashboard (#17494) Bumps [jws](https://github.com/brianloveswords/node-jws) from 3.2.2 to 3.2.3. - [Release notes](https://github.com/brianloveswords/node-jws/releases) - [Changelog](https://github.com/auth0/node-jws/blob/master/CHANGELOG.md) - [Commits](https://github.com/brianloveswords/node-jws/compare/v3.2.2...v3.2.3) --- updated-dependencies: - dependency-name: jws dependency-version: 3.2.3 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/litellm-dashboard/package-lock.json | 50 ++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 9907113c7eb..2ca71c5a8b3 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -324,6 +324,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.5.tgz", "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -2185,6 +2186,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -2227,6 +2229,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -2336,6 +2339,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -2757,6 +2761,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -5824,6 +5829,7 @@ "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", @@ -6617,6 +6623,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.48.tgz", "integrity": "sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w==", "license": "MIT", + "peer": true, "dependencies": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -6639,6 +6646,7 @@ "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", "dev": true, "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^18.0.0" } @@ -6835,6 +6843,7 @@ "integrity": "sha512-lJi3PfxVmo0AkEY93ecfN+r8SofEqZNGByvHAI3GBLrvt1Cw6H5k1IM02nSzu0RfUafr2EvFSw0wAsZgubNplQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.47.0", "@typescript-eslint/types": "8.47.0", @@ -7496,6 +7505,7 @@ "integrity": "sha512-hGISOaP18plkzbWEcP/QvtRW1xDXF2+96HbEX6byqQhAUbiS5oH6/9JwW+QsQCIYON2bI6QZBF+2PvOmrRZ9wA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/utils": "3.2.4", "fflate": "^0.8.2", @@ -7724,6 +7734,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -7813,6 +7824,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -8666,6 +8678,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.25", "caniuse-lite": "^1.0.30001754", @@ -8990,6 +9003,7 @@ "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-11.0.3.tgz", "integrity": "sha512-ci2iJH6LeIkvP9eJW6gpueU8cnZhv85ELY8w8WiFtNjMHA5ad6pQLaJo9mEly/9qUyCpvqX8/POVUTf18/HFdw==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@chevrotain/cst-dts-gen": "11.0.3", "@chevrotain/gast": "11.0.3", @@ -9718,6 +9732,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -10080,6 +10095,7 @@ "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.33.1.tgz", "integrity": "sha512-iJc4TwyANnOGR1OmWhsS9ayRS3s+XQ185FmuHObThD+5AeJCakAAbWv8KimMTt08xCCLNgneQwFp+JRJOr9qGQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10" } @@ -10489,6 +10505,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -10663,6 +10680,7 @@ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-3.6.0.tgz", "integrity": "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==", "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/kossnocorp" @@ -11540,6 +11558,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -11725,6 +11744,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -14988,6 +15008,7 @@ "integrity": "sha512-454TI39PeRDW1LgpyLPyURtB4Zx1tklSr6+OFOipsxGUH1WMTvk6C65JQdrj455+DP2uJ1+veBEHTGFKWVLFoA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@acemir/cssom": "^0.9.23", "@asamuzakjp/dom-selector": "^6.7.4", @@ -15142,12 +15163,12 @@ } }, "node_modules/jws": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", - "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.3.tgz", + "integrity": "sha512-byiJ0FLRdLdSVSReO/U4E7RoEyOCKnEnEPMjq3HxWtvzLsV08/i5RQKsFVNkCldrCaPr2vDNAOMsfs8T/Hze7g==", "license": "MIT", "dependencies": { - "jwa": "^1.4.1", + "jwa": "^1.4.2", "safe-buffer": "^5.0.1" } }, @@ -18115,6 +18136,7 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", "license": "MIT", + "peer": true, "engines": { "node": "*" } @@ -19306,6 +19328,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -20318,6 +20341,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -21812,6 +21836,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -21851,6 +21876,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -21908,6 +21934,7 @@ "resolved": "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-6.0.0.tgz", "integrity": "sha512-YMMxTUQV/QFSnbgrP3tjDzLHRg7vsbMn8e9HAa8o/1iXoiomo48b7sk/kkmWEuWNDPJVlKSJRB6Y2fHqdJk+SQ==", "license": "MIT", + "peer": true, "dependencies": { "@types/react": "*" }, @@ -21973,6 +22000,7 @@ "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.3.4.tgz", "integrity": "sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.12.13", "history": "^4.9.0", @@ -23010,8 +23038,7 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/schema-dts/-/schema-dts-1.1.5.tgz", "integrity": "sha512-RJr9EaCmsLzBX2NDiO5Z3ux2BVosNZN5jo0gWgsyKvxKIUL5R3swNvoorulAeL9kLB0iTSX7V6aokhla2m7xbg==", - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/schema-utils": { "version": "4.3.3", @@ -23037,6 +23064,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -24268,6 +24296,7 @@ "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.18.tgz", "integrity": "sha512-6A2rnmW5xZMdw11LYjhcI5846rt9pbLSabY5XPxo+XWdxwZaFEn47Go4NzFiHu9sNNmr/kXivP1vStfvMaK1GQ==", "license": "MIT", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -24578,6 +24607,7 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -24790,7 +24820,8 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" + "license": "0BSD", + "peer": true }, "node_modules/type-check": { "version": "0.4.0", @@ -24923,6 +24954,7 @@ "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -25465,6 +25497,7 @@ "integrity": "sha512-NL8jTlbo0Tn4dUEXEsUg8KeyG/Lkmc4Fnzb8JXN/Ykm9G4HNImjtABMJgkQoVjOBN/j2WAwDTRytdqJbZsah7w==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", @@ -25581,6 +25614,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -25594,6 +25628,7 @@ "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/chai": "^5.2.2", "@vitest/expect": "3.2.4", @@ -25799,6 +25834,7 @@ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.103.0.tgz", "integrity": "sha512-HU1JOuV1OavsZ+mfigY0j8d1TgQgbZ6M+J75zDkpEAwYeXjWSqrGJtgnPblJjd/mAyTNQ7ygw0MiKOn6etz8yw==", "license": "MIT", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", From 2ffe8ee204723cf9bf8cca7d2d56174c2788bc01 Mon Sep 17 00:00:00 2001 From: Dominic Fallows Date: Fri, 5 Dec 2025 23:45:19 +0000 Subject: [PATCH 16/40] fix(presidio): handle empty content and error dict responses (#17489) - Skip empty/whitespace text before calling Presidio API - Handle error dict responses gracefully (e.g., {'error': 'No text provided'}) - Add defensive error handling for invalid result items - Add comprehensive test coverage for empty content scenarios Fixes crash in tool/function calling where assistant messages have empty content. --- .../guardrails/guardrail_hooks/presidio.py | 43 +++- .../guardrail_hooks/test_presidio.py | 222 ++++++++++++++++++ 2 files changed, 264 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index d183b688edd..8666f6add53 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -207,6 +207,14 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): Send text to the Presidio analyzer endpoint and get analysis results """ try: + # Skip empty or whitespace-only text to avoid Presidio errors + # Common in tool/function calling where assistant content is empty + if not text or len(text.strip()) == 0: + verbose_proxy_logger.debug( + "Skipping Presidio analysis for empty/whitespace-only text" + ) + return [] + async with aiohttp.ClientSession() as session: if self.mock_redacted_text is not None: return self.mock_redacted_text @@ -231,9 +239,42 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): async with session.post(analyze_url, json=analyze_payload) as response: analyze_results = await response.json() verbose_proxy_logger.debug("analyze_results: %s", analyze_results) + + # Handle error responses from Presidio (e.g., {'error': 'No text provided'}) + # Presidio may return a dict instead of a list when errors occur + if isinstance(analyze_results, dict): + if "error" in analyze_results: + verbose_proxy_logger.warning( + "Presidio analyzer returned error: %s, returning empty list", + analyze_results.get("error") + ) + return [] + # If it's a dict but not an error, try to process it as a single item + verbose_proxy_logger.debug( + "Presidio returned dict (not list), attempting to process as single item" + ) + try: + return [PresidioAnalyzeResponseItem(**analyze_results)] + except Exception as e: + verbose_proxy_logger.warning( + "Failed to parse Presidio dict response: %s, returning empty list", + e + ) + return [] + + # Normal case: list of results final_results = [] for item in analyze_results: - final_results.append(PresidioAnalyzeResponseItem(**item)) + try: + final_results.append(PresidioAnalyzeResponseItem(**item)) + except TypeError as te: + # Handle case where item is not a dict (shouldn't happen, but be defensive) + verbose_proxy_logger.warning( + "Skipping invalid Presidio result item: %s (error: %s)", + item, + te + ) + continue return final_results except Exception as e: raise e diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index a6b2ae5b3a0..6450b9a63b0 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -634,6 +634,228 @@ async def test_request_data_flows_to_apply_guardrail(): print("✓ request_data correctly passed to apply_guardrail") +@pytest.mark.asyncio +async def test_empty_content_handling(presidio_guardrail, mock_user_api_key, mock_cache): + """ + Test that Presidio handles empty content gracefully. + + This is common in tool/function calling where assistant messages have + empty content but include tool_calls. + + Bug fix: Previously crashed with: + TypeError: argument after ** must be a mapping, not str + """ + test_data = { + "messages": [ + {"role": "user", "content": "What is 2+2?"}, + { + "role": "assistant", + "content": "", # Empty content - common in tool calls + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": {"name": "calculator", "arguments": '{"a":2,"b":2}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_123", "content": "4"}, + ], + "model": "gpt-4", + } + + # Mock check_pii to simulate PII processing without needing Presidio API + async def mock_check_pii(text, output_parse_pii, presidio_config, request_data): + # Empty text returns as-is (this is what our fix ensures) + return text + + presidio_guardrail.check_pii = mock_check_pii + + # This should not raise an exception + result = await presidio_guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key, + cache=mock_cache, + data=test_data, + call_type="completion", + ) + + assert result is not None + assert "messages" in result + # Verify messages are preserved + assert len(result["messages"]) == 3 + + print("✓ Empty content handling test passed") + + +@pytest.mark.asyncio +async def test_whitespace_only_content(presidio_guardrail, mock_user_api_key, mock_cache): + """ + Test that Presidio handles whitespace-only content gracefully. + + Whitespace-only content should be treated the same as empty content. + """ + test_data = { + "messages": [ + {"role": "user", "content": " "}, # Whitespace only + {"role": "assistant", "content": "\n\t "}, # Tabs and newlines + {"role": "user", "content": "Real question here"}, + ], + "model": "gpt-4", + } + + # Mock check_pii to simulate PII processing + async def mock_check_pii(text, output_parse_pii, presidio_config, request_data): + return text + + presidio_guardrail.check_pii = mock_check_pii + + result = await presidio_guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key, + cache=mock_cache, + data=test_data, + call_type="completion", + ) + + assert result is not None + assert len(result["messages"]) == 3 + + print("✓ Whitespace-only content test passed") + + +@pytest.mark.asyncio +async def test_analyze_text_with_empty_string(): + """ + Test analyze_text method directly with empty string. + + Should return empty list without making API call to Presidio. + """ + presidio = _OPTIONAL_PresidioPIIMasking( + presidio_analyzer_api_base="http://test:5002/", + presidio_anonymizer_api_base="http://test:5001/", + output_parse_pii=False, + ) + + # Test with empty string - should return immediately without API call + result = await presidio.analyze_text( + text="", + presidio_config=None, + request_data={}, + ) + assert result == [], "Empty text should return empty list" + + # Test with whitespace only - should return immediately + result = await presidio.analyze_text( + text=" \n\t ", + presidio_config=None, + request_data={}, + ) + assert result == [], "Whitespace-only text should return empty list" + + print("✓ analyze_text empty string test passed") + + +@pytest.mark.asyncio +async def test_analyze_text_error_dict_handling(): + """ + Test that analyze_text handles error dict responses from Presidio API. + + When Presidio returns {'error': 'No text provided'}, should handle gracefully + instead of crashing with TypeError. + """ + presidio = _OPTIONAL_PresidioPIIMasking( + presidio_analyzer_api_base="http://mock-presidio:5002/", + presidio_anonymizer_api_base="http://mock-presidio:5001/", + output_parse_pii=False, + ) + + # Mock the HTTP response to return error dict + class MockResponse: + async def json(self): + return {"error": "No text provided"} + async def __aenter__(self): + return self + async def __aexit__(self, *args): + pass + + class MockSession: + def post(self, *args, **kwargs): + return MockResponse() + async def __aenter__(self): + return self + async def __aexit__(self, *args): + pass + + with patch("aiohttp.ClientSession", return_value=MockSession()): + result = await presidio.analyze_text( + text="some text", + presidio_config=None, + request_data={}, + ) + # Should return empty list when error dict is received + assert result == [], "Error dict should be handled gracefully" + + print("✓ analyze_text error dict handling test passed") + + +@pytest.mark.asyncio +async def test_tool_calling_complete_scenario(presidio_guardrail, mock_user_api_key, mock_cache): + """ + Test complete tool calling scenario with PII in user message. + + This tests the real-world scenario where: + 1. User provides a query with PII + 2. Assistant responds with empty content + tool_calls + 3. Tool provides response + 4. Assistant provides final answer + """ + test_data = { + "messages": [ + { + "role": "user", + "content": "My email is john.doe@example.com. Can you look up my account?", + }, + { + "role": "assistant", + "content": "", # Empty - tool call + "tool_calls": [ + { + "id": "call_abc", + "type": "function", + "function": {"name": "lookup_account", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_abc", "content": "Account found"}, + {"role": "assistant", "content": "I found your account information."}, + ], + "model": "gpt-4", + } + + # Mock check_pii to simulate PII masking + async def mock_check_pii(text, output_parse_pii, presidio_config, request_data): + if "john.doe@example.com" in text: + return text.replace("john.doe@example.com", "[EMAIL]") + return text + + presidio_guardrail.check_pii = mock_check_pii + + result = await presidio_guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key, + cache=mock_cache, + data=test_data, + call_type="completion", + ) + + assert result is not None + # Verify PII was masked in user message + assert "[EMAIL]" in result["messages"][0]["content"] + assert "john.doe@example.com" not in result["messages"][0]["content"] + # Verify other messages preserved + assert len(result["messages"]) == 4 + + print("✓ Tool calling complete scenario test passed") + + if __name__ == "__main__": # Run tests asyncio.run( From ae065525ea6d331bc36a451fae94319023c638cc Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Fri, 5 Dec 2025 15:39:22 -0800 Subject: [PATCH 17/40] fix ZAI --- litellm/__init__.py | 53 +++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 28 ++++++++++ 2 files changed, 81 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index e312169ffc7..f87dee6ba93 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -520,6 +520,7 @@ perplexity_models: Set = set() watsonx_models: Set = set() gemini_models: Set = set() xai_models: Set = set() +zai_models: Set = set() deepseek_models: Set = set() runwayml_models: Set = set() azure_ai_models: Set = set() @@ -711,6 +712,8 @@ def add_known_models(): text_completion_codestral_models.add(key) elif value.get("litellm_provider") == "xai": xai_models.add(key) + elif value.get("litellm_provider") == "zai": + zai_models.add(key) elif value.get("litellm_provider") == "fal_ai": fal_ai_models.add(key) elif value.get("litellm_provider") == "deepseek": @@ -872,6 +875,7 @@ model_list = list( | gemini_models | text_completion_codestral_models | xai_models + | zai_models | fal_ai_models | deepseek_models | azure_ai_models @@ -960,6 +964,7 @@ models_by_provider: dict = { "aleph_alpha": aleph_alpha_models, "text-completion-codestral": text_completion_codestral_models, "xai": xai_models, + "zai": zai_models, "fal_ai": fal_ai_models, "deepseek": deepseek_models, "runwayml": runwayml_models, @@ -1497,10 +1502,58 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None: # Lazy loading system for heavy modules to reduce initial import time and memory usage if TYPE_CHECKING: + from litellm.types.utils import ModelInfo + + # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] completion_cost: Callable[..., float] response_cost_calculator: Any modify_integration: Any + + # Utils functions - type stubs for lazy loaded functions + exception_type: Callable[..., Any] + get_optional_params: Callable[..., dict] + get_response_string: Callable[..., str] + token_counter: Callable[..., int] + create_pretrained_tokenizer: Callable[..., Any] + create_tokenizer: Callable[..., Any] + supports_function_calling: Callable[..., bool] + supports_web_search: Callable[..., bool] + supports_url_context: Callable[..., bool] + supports_response_schema: Callable[..., bool] + supports_parallel_function_calling: Callable[..., bool] + supports_vision: Callable[..., bool] + supports_audio_input: Callable[..., bool] + supports_audio_output: Callable[..., bool] + supports_system_messages: Callable[..., bool] + supports_reasoning: Callable[..., bool] + get_litellm_params: Callable[..., dict] + acreate: Callable[..., Any] + get_max_tokens: Callable[..., int] + get_model_info: Callable[..., ModelInfo] + register_prompt_template: Callable[..., None] + validate_environment: Callable[..., dict] + check_valid_key: Callable[..., bool] + register_model: Callable[..., None] + encode: Callable[..., list] + decode: Callable[..., str] + _calculate_retry_after: Callable[..., float] + _should_retry: Callable[[int], bool] + get_supported_openai_params: Callable[..., Optional[list]] + get_api_base: Callable[..., Optional[str]] + get_first_chars_messages: Callable[..., str] + get_provider_fields: Callable[..., dict] + get_valid_models: Callable[..., list] + + # Response types - lazy loaded + ModelResponse: Type[Any] + ModelResponseStream: Type[Any] + EmbeddingResponse: Type[Any] + ImageResponse: Type[Any] + TranscriptionResponse: Type[Any] + TextCompletionResponse: Type[Any] + ModelResponseListIterator: Type[Any] + Logging: Type[Any] def __getattr__(name: str) -> Any: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 634ea6dc48a..d4afde20e93 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5164,6 +5164,19 @@ "supports_function_calling": true, "supports_tool_choice": true }, + "azure_ai/mistral-large-3": { + "input_cost_per_token": 5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 256000, + "max_output_tokens": 8191, + "max_tokens": 8191, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://azure.microsoft.com/en-us/blog/introducing-mistral-large-3-in-microsoft-foundry-open-capable-and-ready-for-production-workloads/", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure_ai/mistral-medium-2505": { "input_cost_per_token": 4e-07, "litellm_provider": "azure_ai", @@ -18745,6 +18758,21 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/mistral-large-3": { + "input_cost_per_token": 5e-07, + "litellm_provider": "mistral", + "max_input_tokens": 256000, + "max_output_tokens": 8191, + "max_tokens": 8191, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "mistral/mistral-medium": { "input_cost_per_token": 2.7e-06, "litellm_provider": "mistral", From e519462efab817a3d708d1e9cd47b7cf340d5191 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Fri, 5 Dec 2025 15:46:14 -0800 Subject: [PATCH 18/40] fix MYPY linting --- litellm/__init__.py | 26 +++-------- .../get_llm_provider_logic.py | 4 +- .../chat/guardrail_translation/handler.py | 46 +++++++++---------- litellm/llms/openai_like/dynamic_config.py | 7 ++- .../llms/vertex_ai/gemini/transformation.py | 4 +- .../text_to_speech/transformation.py | 2 +- litellm/proxy/auth/login_utils.py | 14 ++++-- .../generic_guardrail_api.py | 2 +- .../health_endpoints/_health_endpoints.py | 2 +- .../proxy/hooks/key_management_event_hooks.py | 1 - litellm/proxy/proxy_server.py | 22 +++++---- litellm/utils.py | 4 +- 12 files changed, 68 insertions(+), 66 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index f87dee6ba93..10ea521cc9c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1502,7 +1502,7 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None: # Lazy loading system for heavy modules to reduce initial import time and memory usage if TYPE_CHECKING: - from litellm.types.utils import ModelInfo + from litellm.types.utils import ModelInfo as _ModelInfoType # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] @@ -1510,13 +1510,9 @@ if TYPE_CHECKING: response_cost_calculator: Any modify_integration: Any - # Utils functions - type stubs for lazy loaded functions - exception_type: Callable[..., Any] - get_optional_params: Callable[..., dict] + # Utils functions - type stubs for truly lazy loaded functions only + # (functions NOT imported via "from .main import *") get_response_string: Callable[..., str] - token_counter: Callable[..., int] - create_pretrained_tokenizer: Callable[..., Any] - create_tokenizer: Callable[..., Any] supports_function_calling: Callable[..., bool] supports_web_search: Callable[..., bool] supports_url_context: Callable[..., bool] @@ -1527,10 +1523,9 @@ if TYPE_CHECKING: supports_audio_output: Callable[..., bool] supports_system_messages: Callable[..., bool] supports_reasoning: Callable[..., bool] - get_litellm_params: Callable[..., dict] acreate: Callable[..., Any] get_max_tokens: Callable[..., int] - get_model_info: Callable[..., ModelInfo] + get_model_info: Callable[..., _ModelInfoType] register_prompt_template: Callable[..., None] validate_environment: Callable[..., dict] check_valid_key: Callable[..., bool] @@ -1538,22 +1533,15 @@ if TYPE_CHECKING: encode: Callable[..., list] decode: Callable[..., str] _calculate_retry_after: Callable[..., float] - _should_retry: Callable[[int], bool] + _should_retry: Callable[..., bool] get_supported_openai_params: Callable[..., Optional[list]] get_api_base: Callable[..., Optional[str]] get_first_chars_messages: Callable[..., str] - get_provider_fields: Callable[..., dict] + get_provider_fields: Callable[..., List] get_valid_models: Callable[..., list] - # Response types - lazy loaded - ModelResponse: Type[Any] - ModelResponseStream: Type[Any] - EmbeddingResponse: Type[Any] - ImageResponse: Type[Any] - TranscriptionResponse: Type[Any] - TextCompletionResponse: Type[Any] + # Response types - truly lazy loaded only (not in main.py or elsewhere) ModelResponseListIterator: Type[Any] - Logging: Type[Any] def __getattr__(name: str) -> Any: diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 288c122e0e7..ca00370729b 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -469,11 +469,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 model = model.split("/", 1)[1] # Check JSON providers FIRST (before hardcoded ones) - from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry if JSONProviderRegistry.exists(custom_llm_provider): provider_config = JSONProviderRegistry.get(custom_llm_provider) + if provider_config is None: + raise ValueError(f"Provider {custom_llm_provider} not found") config_class = create_config_class(provider_config) api_base, dynamic_api_key = config_class()._get_openai_compatible_provider_info( api_base, api_key diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index aa2580453a8..809c3e4d3e0 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -164,7 +164,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): for tool_call_idx, tool_call in enumerate(tool_calls): if isinstance(tool_call, dict): # Add the full tool call object to the list - tool_calls_to_check.append(ChatCompletionToolParam(**tool_call)) + tool_calls_to_check.append(cast(ChatCompletionToolParam, tool_call)) tool_call_task_mappings.append((msg_idx, int(tool_call_idx))) async def _apply_guardrail_responses_to_input_texts( @@ -380,20 +380,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if isinstance(content, str): # String content - accumulate for this choice - key = (choice_idx, None) - if key not in combined_texts: - combined_texts[key] = "" - combined_texts[key] += content + str_key: Tuple[int, Optional[int]] = (choice_idx, None) + if str_key not in combined_texts: + combined_texts[str_key] = "" + combined_texts[str_key] += content elif isinstance(content, list): # List content - accumulate for each content item for content_idx, content_item in enumerate(content): text_str = content_item.get("text") if text_str: - key = (choice_idx, content_idx) - if key not in combined_texts: - combined_texts[key] = "" - combined_texts[key] += text_str + list_key: Tuple[int, Optional[int]] = (choice_idx, content_idx) + if list_key not in combined_texts: + combined_texts[list_key] = "" + combined_texts[list_key] += text_str # Step 2: Create lists for guardrail processing texts_to_check: List[str] = [] @@ -401,9 +401,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings: List[Tuple[int, Optional[int]]] = [] # Track (choice_index, content_index) for each combined text - for (choice_idx, content_idx), combined_text in combined_texts.items(): + for (map_choice_idx, map_content_idx), combined_text in combined_texts.items(): texts_to_check.append(combined_text) - task_mappings.append((choice_idx, content_idx)) + task_mappings.append((map_choice_idx, map_content_idx)) # Step 3: Apply guardrail to all combined texts in batch if texts_to_check: @@ -503,7 +503,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # Determine content source and tool calls based on choice type content = None - tool_calls = None + tool_calls: Optional[List[Any]] = None if isinstance(choice, litellm.Choices): content = choice.message.content tool_calls = choice.message.tool_calls @@ -686,15 +686,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if isinstance(content, str): # String content - key = (choice_idx_in_response, None) - if key in guardrail_map: - if key not in already_set: + str_key: Tuple[int, Optional[int]] = (choice_idx_in_response, None) + if str_key in guardrail_map: + if str_key not in already_set: # First chunk - set the complete guardrailed text if isinstance(choice, litellm.StreamingChoices): - choice.delta.content = guardrail_map[key] + choice.delta.content = guardrail_map[str_key] elif isinstance(choice, litellm.Choices): - choice.message.content = guardrail_map[key] - already_set[key] = True + choice.message.content = guardrail_map[str_key] + already_set[str_key] = True else: # Subsequent chunks - clear the content if isinstance(choice, litellm.StreamingChoices): @@ -706,12 +706,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # List content - handle each content item for content_idx, content_item in enumerate(content): if "text" in content_item: - key = (choice_idx_in_response, content_idx) - if key in guardrail_map: - if key not in already_set: + list_key: Tuple[int, Optional[int]] = (choice_idx_in_response, content_idx) + if list_key in guardrail_map: + if list_key not in already_set: # First chunk - set the complete guardrailed text - content_item["text"] = guardrail_map[key] - already_set[key] = True + content_item["text"] = guardrail_map[list_key] + already_set[list_key] = True else: # Subsequent chunks - clear the text content_item["text"] = "" diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index ca2489799c2..1e7866bebbe 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -19,11 +19,11 @@ def create_config_class(provider: SimpleProviderConfig): """Generate config class dynamically from JSON configuration""" # Choose base class - base_class = ( + base_class: type = ( OpenAIGPTConfig if provider.base_class == "openai_gpt" else OpenAILikeChatConfig ) - class JSONProviderConfig(base_class): + class JSONProviderConfig(base_class): # type: ignore[valid-type,misc] @overload def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: Literal[True] @@ -87,6 +87,9 @@ def create_config_class(provider: SimpleProviderConfig): if not api_base: api_base = provider.base_url + if api_base is None: + raise ValueError(f"api_base is required for provider {provider.slug}") + if not api_base.endswith("/chat/completions"): api_base = f"{api_base}/chat/completions" diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 3151a6d667e..a95d5447e97 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -116,7 +116,7 @@ def _process_gemini_image( is not None ): file_data = FileDataType(file_uri=image_url, mime_type=image_type) - part: PartType = {"file_data": file_data} + part = {"file_data": file_data} if media_resolution_enum is not None and model is not None: from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig @@ -129,7 +129,7 @@ def _process_gemini_image( image = convert_to_anthropic_image_obj(image_url, format=format) _blob: BlobType = {"data": image["data"], "mime_type": image["media_type"]} - part: PartType = {"inline_data": cast(BlobType, _blob)} + part = {"inline_data": cast(BlobType, _blob)} if media_resolution_enum is not None and model is not None: from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index aff14b1004f..18ca077c4da 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -220,7 +220,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): Returns: Tuple of (mapped_voice_str, mapped_params) """ - mapped_params = {} + mapped_params: Dict[str, Any] = {} ########################################################## # Map voice using helper diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 6ef983b221c..fb9757ca647 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -9,9 +9,9 @@ import os import secrets from typing import Literal, Optional, cast -import litellm from fastapi import HTTPException +import litellm from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.proxy._types import ( LiteLLM_UserTable, @@ -64,13 +64,19 @@ def get_ui_credentials(master_key: Optional[str]) -> tuple[str, str]: class LoginResult: """Result object containing authentication data from login.""" + user_id: str + key: str + user_email: Optional[str] + user_role: str + login_method: Literal["sso", "username_password"] + def __init__( self, user_id: str, key: str, user_email: Optional[str], user_role: str, - login_method: str = "username_password", + login_method: Literal["sso", "username_password"] = "username_password", ): self.user_id = user_id self.key = key @@ -193,14 +199,14 @@ async def authenticate_user( key = response["token"] # type: ignore if get_secret_bool("EXPERIMENTAL_UI_LOGIN"): + from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken + user_info: Optional[LiteLLM_UserTable] = None if _user_row is not None: user_info = _user_row elif ( user_id is not None ): # if user_id is not None, we are using the UI_USERNAME and UI_PASSWORD - from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken - user_info = LiteLLM_UserTable( user_id=user_id, user_role=user_role, diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index c7b4f19a089..6ad21a4758a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -127,7 +127,7 @@ class GenericGuardrailAPI(CustomGuardrail): for field_name in GenericGuardrailAPIMetadata.__annotations__.keys(): value = metadata_dict.get(field_name) if value is not None: - result_metadata[field_name] = value + result_metadata[field_name] = value # type: ignore[literal-required] # handle user_api_key_token = user_api_key_hash if metadata_dict.get("user_api_key_token") is not None: diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 5e4784d709e..62a2aca05dc 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -409,7 +409,7 @@ def _build_model_param_to_info_mapping(model_list: list) -> dict: Returns: Dictionary mapping model parameter to list of model info dicts """ - model_param_to_info = {} + model_param_to_info: dict = {} for model in model_list: model_info = model.get("model_info", {}) model_name = model.get("model_name") diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index 5cfc85ae7aa..3aa62eeeede 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -7,7 +7,6 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.proxy._types import ( - CommonProxyErrors, GenerateKeyRequest, GenerateKeyResponse, KeyRequest, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c1a5fd6c924..2caaec47243 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -45,7 +45,10 @@ from litellm.constants import ( LITELLM_SETTINGS_SAFE_DB_OVERRIDES, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.proxy.common_utils.callback_utils import normalize_callback_names +from litellm.proxy.common_utils.callback_utils import ( + normalize_callback_names, + process_callback, +) from litellm.proxy.common_utils.realtime_utils import _realtime_request_body from litellm.types.utils import ( ModelResponse, @@ -54,7 +57,6 @@ from litellm.types.utils import ( TokenCountResponse, ) from litellm.utils import load_credentials_from_list -from litellm.proxy.common_utils.callback_utils import process_callback if TYPE_CHECKING: from aiohttp import ClientSession @@ -168,8 +170,8 @@ from litellm.constants import ( PROXY_BUDGET_RESCHEDULER_MIN_TIME, ) from litellm.exceptions import RejectedRequestError -from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_litellm_metadata_from_kwargs, @@ -613,7 +615,7 @@ async def proxy_shutdown_event(): await jwt_handler.close() if db_writer_client is not None: - await db_writer_client.close() + await db_writer_client.close() # type: ignore[reportGeneralTypeIssues] # flush remaining langfuse logs if "langfuse" in litellm.success_callback: @@ -792,7 +794,7 @@ async def proxy_startup_event(app: FastAPI): except Exception as e: verbose_proxy_logger.error(f"Error closing shared aiohttp session: {e}") - await proxy_shutdown_event() + await proxy_shutdown_event() # type: ignore[reportGeneralTypeIssues] app = FastAPI( @@ -802,7 +804,7 @@ app = FastAPI( description=_description, version=version, root_path=server_root_path, # check if user passed root path, FastAPI defaults this value to "" - lifespan=proxy_startup_event, + lifespan=proxy_startup_event, # type: ignore[reportGeneralTypeIssues] ) vertex_live_passthrough_vertex_base = VertexBase() @@ -8330,9 +8332,9 @@ async def login(request: Request): # noqa: PLR0915 # Generate JWT token import jwt - jwt_token = jwt.encode( # type: ignore + jwt_token = jwt.encode( cast(dict, returned_ui_token_object), - master_key, + cast(str, master_key), algorithm="HS256", ) @@ -8377,9 +8379,9 @@ async def login_v2(request: Request): # noqa: PLR0915 import jwt - jwt_token = jwt.encode( # type: ignore + jwt_token = jwt.encode( cast(dict, returned_ui_token_object), - master_key, + cast(str, master_key), algorithm="HS256", ) diff --git a/litellm/utils.py b/litellm/utils.py index 0db84d3f5b9..1d10ecce016 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7023,11 +7023,13 @@ class ProviderConfigManager: """ # Check JSON providers FIRST - from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry if JSONProviderRegistry.exists(provider.value): provider_config = JSONProviderRegistry.get(provider.value) + if provider_config is None: + raise ValueError(f"Provider {provider.value} not found") return create_config_class(provider_config)() if ( From bffc1181709a5ebfa8fd6d4f230d1c1e4adf30a1 Mon Sep 17 00:00:00 2001 From: Irfan Sofyana Putra Date: Sat, 6 Dec 2025 06:47:34 +0700 Subject: [PATCH 19/40] fix bedrock qwen anthropic beta (#17467) --- .../bedrock/chat/converse_transformation.py | 5 +- .../bedrock/test_anthropic_beta_support.py | 100 ++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 705f3c9e630..2a1d7f2e3a3 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -960,7 +960,10 @@ class AmazonConverseConfig(BaseConfig): bedrock_tools = _bedrock_tools_pt(filtered_tools) # Set anthropic_beta in additional_request_params if we have any beta features - if anthropic_beta_list: + # ONLY apply to Anthropic/Claude models - other models (e.g., Qwen, Llama) don't support this field + # and will error with "unknown variant anthropic_beta" if included + base_model = BedrockModelInfo.get_base_model(model) + if anthropic_beta_list and base_model.startswith("anthropic"): # Remove duplicates while preserving order unique_betas = [] seen = set() diff --git a/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py b/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py index 7de2294954c..b9324e4966f 100644 --- a/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py +++ b/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py @@ -290,3 +290,103 @@ class TestAnthropicBetaHeaderSupport: else: # If no beta headers, that's also fine assert True + + def test_converse_non_anthropic_model_no_anthropic_beta(self): + """Test that non-Anthropic models (e.g., Qwen) do NOT get anthropic_beta in additionalModelRequestFields. + + This is critical because non-Anthropic models on Bedrock will error with + "unknown variant anthropic_beta" if this field is included. + """ + config = AmazonConverseConfig() + # Even if headers contain anthropic-beta, non-Anthropic models should NOT get it + headers = {"anthropic-beta": "context-1m-2025-08-07,interleaved-thinking-2025-05-14"} + + # Test with Qwen model (using ARN format like the user's config) + result = config._transform_request_helper( + model="qwen.qwen3-coder-480b-a35b-v1:0", + system_content_blocks=[], + optional_params={}, + messages=[{"role": "user", "content": "Test"}], + headers=headers + ) + + additional_fields = result.get("additionalModelRequestFields", {}) + assert "anthropic_beta" not in additional_fields, ( + "anthropic_beta should NOT be added for non-Anthropic models like Qwen. " + "This field is only supported by Anthropic/Claude models on Bedrock." + ) + + def test_converse_llama_model_no_anthropic_beta(self): + """Test that Llama models do NOT get anthropic_beta in additionalModelRequestFields.""" + config = AmazonConverseConfig() + headers = {"anthropic-beta": "context-1m-2025-08-07"} + + result = config._transform_request_helper( + model="meta.llama3-2-11b-instruct-v1:0", + system_content_blocks=[], + optional_params={}, + messages=[{"role": "user", "content": "Test"}], + headers=headers + ) + + additional_fields = result.get("additionalModelRequestFields", {}) + assert "anthropic_beta" not in additional_fields, ( + "anthropic_beta should NOT be added for Llama models." + ) + + def test_converse_nova_model_no_anthropic_beta(self): + """Test that Amazon Nova models do NOT get anthropic_beta in additionalModelRequestFields.""" + config = AmazonConverseConfig() + headers = {"anthropic-beta": "computer-use-2024-10-22"} + + result = config._transform_request_helper( + model="amazon.nova-pro-v1:0", + system_content_blocks=[], + optional_params={}, + messages=[{"role": "user", "content": "Test"}], + headers=headers + ) + + additional_fields = result.get("additionalModelRequestFields", {}) + assert "anthropic_beta" not in additional_fields, ( + "anthropic_beta should NOT be added for Amazon Nova models." + ) + + def test_converse_anthropic_model_gets_anthropic_beta(self): + """Test that Anthropic models DO get anthropic_beta in additionalModelRequestFields.""" + config = AmazonConverseConfig() + headers = {"anthropic-beta": "context-1m-2025-08-07"} + + result = config._transform_request_helper( + model="anthropic.claude-3-5-sonnet-20241022-v2:0", + system_content_blocks=[], + optional_params={}, + messages=[{"role": "user", "content": "Test"}], + headers=headers + ) + + additional_fields = result.get("additionalModelRequestFields", {}) + assert "anthropic_beta" in additional_fields, ( + "anthropic_beta SHOULD be added for Anthropic models." + ) + assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"] + + def test_converse_anthropic_model_with_cross_region_prefix(self): + """Test that Anthropic models with cross-region prefix still get anthropic_beta.""" + config = AmazonConverseConfig() + headers = {"anthropic-beta": "context-1m-2025-08-07"} + + # Model with 'us.' cross-region prefix + result = config._transform_request_helper( + model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + system_content_blocks=[], + optional_params={}, + messages=[{"role": "user", "content": "Test"}], + headers=headers + ) + + additional_fields = result.get("additionalModelRequestFields", {}) + assert "anthropic_beta" in additional_fields, ( + "anthropic_beta SHOULD be added for Anthropic models with cross-region prefix." + ) + assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"] From 2cf41d63a6eaa5abcfc7ba4f91c33e6b971ae21a Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Fri, 5 Dec 2025 20:51:51 -0300 Subject: [PATCH 20/40] fix(gemini): use thought:true instead of thoughtSignature to detect thinking blocks (#17266) The previous implementation incorrectly used `thoughtSignature` as the criterion to detect thinking blocks. However, per Google's docs: - `thought: true` indicates that a part contains reasoning/thinking content - `thoughtSignature` is just a token for multi-turn context preservation (a part can have thoughtSignature without thought:true, e.g., function calls) This caused functionCall data to leak into reasoning_content when using Gemini 2.5 Pro with streaming + tools enabled. Changes: - _extract_thinking_blocks_from_parts now checks `part.get("thought") is True` - Extract actual text content instead of json.dumps(part) - Include signature only when present (optional in Gemini 2.5) Refs: - https://ai.google.dev/gemini-api/docs/thinking - https://ai.google.dev/gemini-api/docs/thought-signatures --- .../vertex_and_google_ai_studio_gemini.py | 34 +++++---- ...test_vertex_and_google_ai_studio_gemini.py | 73 +++++++++++++++++-- 2 files changed, 83 insertions(+), 24 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index e604bd392a6..106074811f6 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1085,24 +1085,26 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): def _extract_thinking_blocks_from_parts( self, parts: List[HttpxPartType] ) -> List[ChatCompletionThinkingBlock]: - """Extract thinking blocks from parts if present""" + """Extract thinking blocks from parts if present. + + Per Google's docs (https://ai.google.dev/gemini-api/docs/thinking): + - Parts with `thought: true` contain thinking/reasoning content + - `thoughtSignature` is a separate token for multi-turn context preservation, + it does NOT indicate that the content is thinking (a part can have + thoughtSignature without thought: true, e.g., function calls) + """ thinking_blocks: List[ChatCompletionThinkingBlock] = [] for part in parts: - if "thoughtSignature" in part: - part_copy = part.copy() - part_copy.pop("thoughtSignature") - - text_content = part_copy.get("text") - if isinstance(text_content, str) and text_content.strip() == "": - continue - - thinking_blocks.append( - ChatCompletionThinkingBlock( - type="thinking", - thinking=json.dumps(part_copy), - signature=part["thoughtSignature"], - ) - ) + if part.get("thought") is True: + thinking_text = part.get("text", "") + block: ChatCompletionThinkingBlock = { + "type": "thinking", + "thinking": thinking_text, + } + signature = part.get("thoughtSignature") + if signature is not None: + block["signature"] = signature + thinking_blocks.append(block) return thinking_blocks def _extract_image_response_from_parts( diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 41afec9cd14..7d45ce4091a 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -390,13 +390,13 @@ def test_streaming_chunk_includes_reasoning_content(): ) -def test_streaming_chunk_with_tool_calls_includes_reasoning_content(): +def test_streaming_chunk_with_tool_calls_and_thought_includes_reasoning_content(): """ - Test for issue #16805: Ensure that when Gemini returns a streaming chunk with - tool calls AND thoughtSignature, the reasoning_content is included in the delta. + Test that when Gemini returns a streaming chunk with both thought: true parts + AND tool calls, the reasoning_content is correctly extracted from the thought parts. - Previously, thinking_blocks were only added to non-streaming responses, causing - reasoning_content to be missing in streaming mode when tools were enabled. + Per Google's docs: thought: true indicates reasoning content, NOT thoughtSignature. + thoughtSignature is just a token for multi-turn context preservation. """ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( ModelResponseIterator, @@ -409,12 +409,16 @@ def test_streaming_chunk_with_tool_calls_includes_reasoning_content(): { "content": { "parts": [ + { + "text": "Let me think about how to get the time...", + "thought": True, # This indicates reasoning content + }, { "functionCall": { "name": "get_current_time", "args": {"timezone": "America/New_York"}, }, - "thoughtSignature": "EsEDCr4DAdHtim...", # Base64 signature + "thoughtSignature": "EsEDCr4DAdHtim...", # Just a token, not reasoning } ] }, @@ -433,8 +437,8 @@ def test_streaming_chunk_with_tool_calls_includes_reasoning_content(): ) streaming_chunk = iterator.chunk_parser(chunk) - # Verify that reasoning_content is present in the streaming delta - assert streaming_chunk.choices[0].delta.reasoning_content is not None + # Verify reasoning_content comes from the thought: true part + assert streaming_chunk.choices[0].delta.reasoning_content == "Let me think about how to get the time..." # Verify tool calls are also present assert streaming_chunk.choices[0].delta.tool_calls is not None @@ -442,6 +446,59 @@ def test_streaming_chunk_with_tool_calls_includes_reasoning_content(): assert streaming_chunk.choices[0].delta.tool_calls[0].function.name == "get_current_time" +def test_streaming_chunk_with_tool_calls_no_thought_no_reasoning_content(): + """ + Test that when Gemini returns tool calls with thoughtSignature but WITHOUT + thought: true, there is NO reasoning_content. + + This is a regression test for the bug where functionCall data was incorrectly + being placed into reasoning_content when thoughtSignature was present. + Per Google's docs: thoughtSignature is just a token for multi-turn, not reasoning. + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + ModelResponseIterator, + ) + + litellm_logging = MagicMock() + + chunk = { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "get_current_time", + "args": {"timezone": "America/New_York"}, + }, + "thoughtSignature": "EsEDCr4DAdHtim...", # Just a token, NOT thought: true + } + ] + }, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "promptTokenCount": 68, + "candidatesTokenCount": 120, + "totalTokenCount": 188, + }, + } + + iterator = ModelResponseIterator( + streaming_response=[], sync_stream=True, logging_obj=litellm_logging + ) + streaming_chunk = iterator.chunk_parser(chunk) + + # reasoning_content should be None - thoughtSignature alone does NOT mean reasoning + assert getattr(streaming_chunk.choices[0].delta, 'reasoning_content', None) is None + + # Tool calls should still work + assert streaming_chunk.choices[0].delta.tool_calls is not None + assert len(streaming_chunk.choices[0].delta.tool_calls) == 1 + assert streaming_chunk.choices[0].delta.tool_calls[0].function.name == "get_current_time" + + def test_check_finish_reason(): finish_reason_mappings = VertexGeminiConfig.get_finish_reason_mapping() for k, v in finish_reason_mappings.items(): From 852a1fee89754d5f095e2a6e4915bfbc58a1eda3 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 5 Dec 2025 15:51:56 -0800 Subject: [PATCH 21/40] Support images in compare UI --- .../playground/compareUI/CompareUI.test.tsx | 69 ++++++++- .../playground/compareUI/CompareUI.tsx | 136 ++++++++++++++---- .../components/MessageDisplay.test.tsx | 29 ++++ .../compareUI/components/MessageDisplay.tsx | 4 +- .../components/MessageInput.test.tsx | 12 ++ .../compareUI/components/MessageInput.tsx | 11 +- 6 files changed, 230 insertions(+), 31 deletions(-) diff --git a/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.test.tsx b/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.test.tsx index 1941943a58b..963b7527745 100644 --- a/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.test.tsx +++ b/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.test.tsx @@ -2,6 +2,7 @@ import { render, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import CompareUI from "./CompareUI"; +import { makeOpenAIChatCompletionRequest } from "../llm_calls/chat_completion"; vi.mock("../llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-4" }, { model_group: "gpt-3.5-turbo" }]), @@ -11,6 +12,34 @@ vi.mock("../llm_calls/chat_completion", () => ({ makeOpenAIChatCompletionRequest: vi.fn().mockResolvedValue(undefined), })); +let capturedOnImageUpload: ((file: File) => false) | null = null; + +vi.mock("../chat_ui/ChatImageUpload", () => ({ + default: ({ onImageUpload }: { onImageUpload: (file: File) => false }) => { + capturedOnImageUpload = onImageUpload; + return ( +
+ +
+ ); + }, +})); + +vi.mock("../chat_ui/ChatImageUtils", () => ({ + createChatMultimodalMessage: vi.fn().mockResolvedValue({ + role: "user", + content: [ + { type: "text", text: "test message" }, + { type: "image_url", image_url: { url: "data:image/png;base64,test" } }, + ], + }), + createChatDisplayMessage: vi.fn().mockReturnValue({ + role: "user", + content: "test message [Image attached]", + imagePreviewUrl: "blob:test-url", + }), +})); + vi.mock("./components/ComparisonPanel", () => ({ ComparisonPanel: ({ comparison, onRemove }: { comparison: any; onRemove: () => void }) => (
@@ -22,8 +51,9 @@ vi.mock("./components/ComparisonPanel", () => ({ })); vi.mock("./components/MessageInput", () => ({ - MessageInput: ({ value, onChange, onSend, disabled }: any) => ( + MessageInput: ({ value, onChange, onSend, disabled, hasAttachment, uploadComponent }: any) => (
+ {uploadComponent &&
{uploadComponent}
}