From 8ae7d5ce32cb575fad9a90e406964ac901cd9bde Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 17 Dec 2025 21:52:51 -0800 Subject: [PATCH 01/85] base commit --- litellm/proxy/vector_store_endpoints/management_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index fdb1dba372f..b0ddb5c433f 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -48,7 +48,7 @@ async def new_vector_store( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - Create a new vector store. + Create a new vector store Parameters: - vector_store_id: str - Unique identifier for the vector store From 98c9037ab8dc55423463238f4ac36071c234cecb Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 17 Dec 2025 21:56:47 -0800 Subject: [PATCH 02/85] resolve embeddings model config --- .../management_endpoints.py | 115 +++++++++++++++++- 1 file changed, 113 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index b0ddb5c433f..661f94e5f04 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -10,7 +10,7 @@ All /vector_store management endpoints import copy import json -from typing import List, Optional +from typing import Any, Dict, List, Optional from fastapi import APIRouter, Depends, HTTPException @@ -23,6 +23,8 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper +from litellm.secret_managers.main import get_secret from litellm.types.vector_stores import ( LiteLLM_ManagedVectorStore, LiteLLM_ManagedVectorStoreListResponse, @@ -35,6 +37,102 @@ from litellm.vector_stores.vector_store_registry import VectorStoreRegistry router = APIRouter() +async def _resolve_embedding_config_from_db( + embedding_model: str, prisma_client +) -> Optional[Dict[str, Any]]: + """ + Resolve embedding config from database model configuration. + + If litellm_embedding_model is provided but litellm_embedding_config is not, + this function looks up the model in the database and extracts api_key, api_base, + and api_version from the model's litellm_params to build the embedding config. + + Args: + embedding_model: The embedding model string (e.g., "text-embedding-ada-002" or "azure/text-embedding-3-large") + prisma_client: The Prisma client instance + + Returns: + Dictionary with api_key, api_base, and api_version if model found, None otherwise + """ + if not embedding_model: + return None + + # Extract model name - could be "text-embedding-ada-002" or "azure/text-embedding-3-large" + # Try to find model by exact match first, then try without provider prefix + model_name_candidates = [embedding_model] + if "/" in embedding_model: + # If it has a provider prefix, also try without it + _, model_name = embedding_model.split("/", 1) + model_name_candidates.append(model_name) + + # Try to find model in database + for model_name in model_name_candidates: + try: + db_model = await prisma_client.db.litellm_proxymodeltable.find_first( + where={"model_name": model_name} + ) + + if db_model and db_model.litellm_params: + # Extract litellm_params (could be dict or JSON string) + model_params = db_model.litellm_params + if isinstance(model_params, str): + model_params = json.loads(model_params) + + # Decrypt values from database (similar to how proxy_server.py does it) + # Values stored in DB are encrypted, so we need to decrypt them first + decrypted_params = {} + if isinstance(model_params, dict): + for k, v in model_params.items(): + if isinstance(v, str): + # Decrypt value - returns original value if decryption fails or no key is set + decrypted_value = decrypt_value_helper( + value=v, key=k, return_original_value=True + ) + decrypted_params[k] = decrypted_value + else: + decrypted_params[k] = v + else: + decrypted_params = model_params + + # Build embedding config from model params + embedding_config = {} + + # Extract api_key + api_key = decrypted_params.get("api_key") + if api_key: + # Handle os.environ/ prefix (after decryption, values may be os.environ/ prefixed) + if isinstance(api_key, str) and api_key.startswith("os.environ/"): + api_key = get_secret(api_key) + embedding_config["api_key"] = api_key + + # Extract api_base + api_base = decrypted_params.get("api_base") + if api_base: + # Handle os.environ/ prefix (after decryption, values may be os.environ/ prefixed) + if isinstance(api_base, str) and api_base.startswith("os.environ/"): + api_base = get_secret(api_base) + embedding_config["api_base"] = api_base + + # Extract api_version + api_version = decrypted_params.get("api_version") + if api_version: + embedding_config["api_version"] = api_version + + # Only return config if we have at least api_key or api_base + if embedding_config: + verbose_proxy_logger.debug( + f"Resolved embedding config from database model {model_name}: {list(embedding_config.keys())}" + ) + return embedding_config + except Exception as e: + verbose_proxy_logger.debug( + f"Error resolving embedding config for model {model_name}: {str(e)}" + ) + continue + + return None + + ######################################################## # Management Endpoints ######################################################## @@ -48,7 +146,7 @@ async def new_vector_store( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - Create a new vector store + Create a new vector store. Parameters: - vector_store_id: str - Unique identifier for the vector store @@ -85,6 +183,19 @@ async def new_vector_store( litellm_params_json: Optional[str] = None _input_litellm_params: dict = vector_store.get("litellm_params", {}) or {} if _input_litellm_params is not None: + # Auto-resolve embedding config if embedding model is provided but config is not + embedding_model = _input_litellm_params.get("litellm_embedding_model") + if embedding_model and not _input_litellm_params.get("litellm_embedding_config"): + resolved_config = await _resolve_embedding_config_from_db( + embedding_model=embedding_model, + prisma_client=prisma_client + ) + if resolved_config: + _input_litellm_params["litellm_embedding_config"] = resolved_config + verbose_proxy_logger.info( + f"Auto-resolved embedding config for model {embedding_model}" + ) + litellm_params_dict = GenericLiteLLMParams( **_input_litellm_params ).model_dump(exclude_none=True) From 8ea7688d353b79e001421ee0b3e6dd7ba976b902 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 17 Dec 2025 21:59:28 -0800 Subject: [PATCH 03/85] Adding tests --- .../test_vector_store_endpoints.py | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index b98354032fe..6677b2f8436 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -20,6 +20,10 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.vector_store_endpoints.endpoints import ( _update_request_data_with_litellm_managed_vector_store_registry, ) +from litellm.proxy.vector_store_endpoints.management_endpoints import ( + _resolve_embedding_config_from_db, + new_vector_store, +) from litellm.proxy.vector_store_endpoints.utils import ( check_vector_store_permission, is_allowed_to_call_vector_store_endpoint, @@ -1045,3 +1049,138 @@ async def test_vector_store_synchronization_across_instances(): assert len(vector_stores_to_run) == 0, ( "Deleted vector store should not be returned when trying to use it" ) + + +@pytest.mark.asyncio +async def test_resolve_embedding_config_from_db(): + """Test that _resolve_embedding_config_from_db correctly resolves embedding config from database.""" + mock_prisma_client = MagicMock() + + # Mock database model with litellm_params + mock_db_model = MagicMock() + mock_db_model.litellm_params = { + "api_key": "test-api-key", + "api_base": "https://api.openai.com", + "api_version": "2024-01-01" + } + + mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( + return_value=mock_db_model + ) + + with patch( + "litellm.proxy.vector_store_endpoints.management_endpoints.decrypt_value_helper", + side_effect=lambda value, key, return_original_value: value + ): + result = await _resolve_embedding_config_from_db( + embedding_model="text-embedding-ada-002", + prisma_client=mock_prisma_client + ) + + assert result is not None + assert result["api_key"] == "test-api-key" + assert result["api_base"] == "https://api.openai.com" + assert result["api_version"] == "2024-01-01" + mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_called_once_with( + where={"model_name": "text-embedding-ada-002"} + ) + + # Test with empty embedding_model + result_empty = await _resolve_embedding_config_from_db( + embedding_model="", + prisma_client=mock_prisma_client + ) + assert result_empty is None + + # Test with model not found + mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( + return_value=None + ) + result_not_found = await _resolve_embedding_config_from_db( + embedding_model="non-existent-model", + prisma_client=mock_prisma_client + ) + assert result_not_found is None + + +@pytest.mark.asyncio +async def test_new_vector_store_auto_resolves_embedding_config(): + """Test that new_vector_store auto-resolves embedding config when embedding_model is provided but config is not.""" + import json + from litellm.types.vector_stores import LiteLLM_ManagedVectorStore + + mock_prisma_client = MagicMock() + + # Mock vector store request with embedding_model but no embedding_config + vector_store_data: LiteLLM_ManagedVectorStore = { + "vector_store_id": "test-store-001", + "custom_llm_provider": "openai", + "litellm_params": { + "litellm_embedding_model": "text-embedding-ada-002", + # Note: litellm_embedding_config is not provided + } + } + + # Mock database model lookup for embedding config resolution + mock_db_model = MagicMock() + mock_db_model.litellm_params = { + "api_key": "resolved-api-key", + "api_base": "https://api.openai.com", + "api_version": "2024-01-01" + } + + # Mock user API key + mock_user_api_key = MagicMock(spec=UserAPIKeyAuth) + mock_user_api_key.user_role = None + + # Mock database operations + mock_prisma_client.db.litellm_managedvectorstorestable.find_unique = AsyncMock( + return_value=None # Vector store doesn't exist yet + ) + mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( + return_value=mock_db_model + ) + + # Track what was passed to create + captured_create_data = {} + + async def mock_create(*args, **kwargs): + captured_create_data.update(kwargs.get("data", {})) + mock_created_vector_store = MagicMock() + mock_created_vector_store.model_dump.return_value = { + "vector_store_id": "test-store-001", + "custom_llm_provider": "openai", + "litellm_params": kwargs.get("data", {}).get("litellm_params") + } + return mock_created_vector_store + + mock_prisma_client.db.litellm_managedvectorstorestable.create = AsyncMock( + side_effect=mock_create + ) + + mock_registry = MagicMock() + mock_registry.add_vector_store_to_registry = MagicMock() + + with patch( + "litellm.proxy.proxy_server.prisma_client", + mock_prisma_client + ), patch( + "litellm.proxy.vector_store_endpoints.management_endpoints.decrypt_value_helper", + side_effect=lambda value, key, return_original_value: value + ), patch.object( + litellm, "vector_store_registry", mock_registry + ): + result = await new_vector_store( + vector_store=vector_store_data, + user_api_key_dict=mock_user_api_key + ) + + assert result["status"] == "success" + # Verify that embedding config was resolved and included in the create call + litellm_params_json = captured_create_data.get("litellm_params") + assert litellm_params_json is not None + litellm_params_dict = json.loads(litellm_params_json) + assert "litellm_embedding_config" in litellm_params_dict + assert litellm_params_dict["litellm_embedding_config"]["api_key"] == "resolved-api-key" + assert litellm_params_dict["litellm_embedding_config"]["api_base"] == "https://api.openai.com" + assert litellm_params_dict["litellm_embedding_config"]["api_version"] == "2024-01-01" From c071bfbe58e90ce2e171eefd6efff7c956991e83 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 17 Dec 2025 22:05:47 -0800 Subject: [PATCH 04/85] base commit --- litellm/proxy/spend_tracking/cloudzero_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/spend_tracking/cloudzero_endpoints.py b/litellm/proxy/spend_tracking/cloudzero_endpoints.py index 502537cb70f..7aecf7a6451 100644 --- a/litellm/proxy/spend_tracking/cloudzero_endpoints.py +++ b/litellm/proxy/spend_tracking/cloudzero_endpoints.py @@ -296,7 +296,7 @@ def is_cloudzero_setup_in_config() -> bool: async def is_cloudzero_setup() -> bool: """ - Check if CloudZero is setup in either config.yaml/env vars OR database. + Check if CloudZero is setup in either config.yaml/env vars OR database CloudZero is considered setup if: - CloudZero is configured in config.yaml callbacks, OR From b0d371d864ecb9b8680fc0f2f618db63a41f56d5 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 17 Dec 2025 22:12:31 -0800 Subject: [PATCH 05/85] delete route for cloudzero --- .../spend_tracking/cloudzero_endpoints.py | 69 +++++++++++++++- .../test_cloudzero_endpoints.py | 79 +++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py diff --git a/litellm/proxy/spend_tracking/cloudzero_endpoints.py b/litellm/proxy/spend_tracking/cloudzero_endpoints.py index 7aecf7a6451..2cf4ce8f16a 100644 --- a/litellm/proxy/spend_tracking/cloudzero_endpoints.py +++ b/litellm/proxy/spend_tracking/cloudzero_endpoints.py @@ -296,7 +296,7 @@ def is_cloudzero_setup_in_config() -> bool: async def is_cloudzero_setup() -> bool: """ - Check if CloudZero is setup in either config.yaml/env vars OR database + Check if CloudZero is setup in either config.yaml/env vars OR database. CloudZero is considered setup if: - CloudZero is configured in config.yaml callbacks, OR @@ -500,3 +500,70 @@ async def cloudzero_export( status_code=500, detail={"error": f"Failed to perform CloudZero export: {str(e)}"}, ) + + +@router.delete( + "/cloudzero/delete", + tags=["CloudZero"], + dependencies=[Depends(user_api_key_auth)], + response_model=CloudZeroInitResponse, +) +async def delete_cloudzero_settings( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Delete CloudZero settings from the database. + + This endpoint removes the CloudZero configuration (API key, connection ID, timezone) + from the proxy database. Only the CloudZero settings entry will be deleted; + other configuration values in the database will remain unchanged. + + Only admin users can delete CloudZero settings. + """ + # Validation + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail={"error": CommonProxyErrors.not_allowed_access.value}, + ) + + try: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + # Check if CloudZero settings exist + cloudzero_config = await prisma_client.db.litellm_config.find_first( + where={"param_name": "cloudzero_settings"} + ) + + if cloudzero_config is None: + raise HTTPException( + status_code=404, + detail={"error": "CloudZero settings not found"}, + ) + + # Delete only the CloudZero settings entry + # This uses a specific where clause to target only the cloudzero_settings row + await prisma_client.db.litellm_config.delete( + where={"param_name": "cloudzero_settings"} + ) + + verbose_proxy_logger.info("CloudZero settings deleted successfully") + + return CloudZeroInitResponse( + message="CloudZero settings deleted successfully", status="success" + ) + + except HTTPException as e: + raise e + except Exception as e: + verbose_proxy_logger.error(f"Error deleting CloudZero settings: {str(e)}") + raise HTTPException( + status_code=500, + detail={"error": f"Failed to delete CloudZero settings: {str(e)}"}, + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py new file mode 100644 index 00000000000..8ff5774bf50 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py @@ -0,0 +1,79 @@ +import os +import sys +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../../..") +) + +import litellm.proxy.proxy_server as ps +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.proxy_server import app + + +@pytest.fixture +def client(): + return TestClient(app) + + +@pytest.mark.asyncio +async def test_delete_cloudzero_settings_success(client, monkeypatch): + mock_config = MagicMock() + mock_config.param_name = "cloudzero_settings" + mock_config.param_value = {"api_key": "encrypted_key", "connection_id": "conn_123", "timezone": "UTC"} + + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=mock_config) + mock_litellm_config.delete = AsyncMock(return_value=mock_config) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.delete("/cloudzero/delete") + assert response.status_code == 200 + data = response.json() + assert data["message"] == "CloudZero settings deleted successfully" + assert data["status"] == "success" + mock_litellm_config.find_first.assert_awaited_once() + mock_litellm_config.delete.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_delete_cloudzero_settings_not_found(client, monkeypatch): + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=None) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.delete("/cloudzero/delete") + assert response.status_code == 404 + data = response.json() + assert "error" in data["detail"] + assert "CloudZero settings not found" in data["detail"]["error"] + mock_litellm_config.find_first.assert_awaited_once() + mock_litellm_config.delete.assert_not_called() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + From 0a4a7408c7542671f4f433ed8fd14cc991f6a435 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 17 Dec 2025 22:32:39 -0800 Subject: [PATCH 06/85] UI Cloudzero delete route --- .../hooks/cloudzero/useCloudZeroSettings.ts | 51 +++++++++++++- .../CloudZeroIntegrationSettings.test.tsx | 15 +++- .../CloudZeroIntegrationSettings.tsx | 69 ++++++++++++++----- 3 files changed, 114 insertions(+), 21 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts index 2ef23e28247..5ccbe244e60 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts @@ -1,7 +1,7 @@ -import { getProxyBaseUrl } from "@/components/networking"; -import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"; -import { createQueryKeys } from "../common/queryKeysFactory"; import { CloudZeroSettings } from "@/components/CloudZeroCostTracking/types"; +import { getProxyBaseUrl } from "@/components/networking"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; const cloudZeroSettingsKeys = createQueryKeys("cloudZeroSettings"); @@ -54,6 +54,11 @@ interface UpdateResponse { status: string; } +interface DeleteResponse { + message: string; + status: string; +} + const updateCloudZeroSettings = async (accessToken: string, params: UpdateParams): Promise => { const proxyBaseUrl = getProxyBaseUrl(); const url = proxyBaseUrl ? `${proxyBaseUrl}/cloudzero/settings` : `/cloudzero/settings`; @@ -98,3 +103,43 @@ export const useCloudZeroUpdateSettings = (accessToken: string) => { }, }); }; + +const deleteCloudZeroSettings = async (accessToken: string): Promise => { + const proxyBaseUrl = getProxyBaseUrl(); + const url = proxyBaseUrl ? `${proxyBaseUrl}/cloudzero/delete` : `/cloudzero/delete`; + + const response = await fetch(url, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + const errorMessage = + errorData?.error?.message || errorData?.message || errorData?.detail || "Failed to delete CloudZero settings"; + throw new Error(errorMessage); + } + + const data = await response.json(); + return data; +}; + +export const useCloudZeroDeleteSettings = (accessToken: string) => { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: async () => { + if (!accessToken) { + throw new Error("Access token is required"); + } + return await deleteCloudZeroSettings(accessToken); + }, + onSuccess: () => { + // Invalidate the settings query to refetch updated data + queryClient.invalidateQueries({ queryKey: cloudZeroSettingsKeys.list({}) }); + }, + }); +}; diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.test.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.test.tsx index 350c0a3c4da..51179f4014f 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.test.tsx @@ -1,6 +1,6 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; -import { render, screen } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { CloudZeroIntegrationSettings } from "./CloudZeroIntegrationSettings"; import { CloudZeroSettings } from "./types"; @@ -68,4 +68,15 @@ describe("CloudZeroIntegrationSettings", () => { expect(screen.getByText("Connection ID")).toBeInTheDocument(); expect(screen.getByText("Timezone")).toBeInTheDocument(); }); + + it("should display the correct values from settings", () => { + render( + + + , + ); + + expect(screen.getByText(mockSettings.api_key_masked)).toBeInTheDocument(); + expect(screen.getByText(mockSettings.connection_id)).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx index 6afe3d65f83..780fa83652a 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx @@ -1,6 +1,8 @@ import { useCloudZeroDryRun } from "@/app/(dashboard)/hooks/cloudzero/useCloudZeroDryRun"; import { useCloudZeroExport } from "@/app/(dashboard)/hooks/cloudzero/useCloudZeroExport"; +import { useCloudZeroDeleteSettings } from "@/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; import { Alert, Button, Card, Descriptions, Divider, message, Popconfirm, Tag } from "antd"; import { CheckCircle, Edit, Play, Trash2, Upload } from "lucide-react"; import { useState } from "react"; @@ -15,9 +17,11 @@ interface CloudZeroIntegrationSettingsProps { export function CloudZeroIntegrationSettings({ settings, onSettingsUpdated }: CloudZeroIntegrationSettingsProps) { const { accessToken } = useAuthorized(); const [isEditModalOpen, setIsEditModalOpen] = useState(false); + const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const dryRunMutation = useCloudZeroDryRun(accessToken || ""); const exportMutation = useCloudZeroExport(accessToken || ""); + const deleteMutation = useCloudZeroDeleteSettings(accessToken || ""); const handleDryRun = () => { if (!accessToken) return; @@ -66,10 +70,27 @@ export function CloudZeroIntegrationSettings({ settings, onSettingsUpdated }: Cl setIsEditModalOpen(false); }; - const handleDelete = async () => { - // Note: Delete functionality is not yet implemented in the backend API - // This would require a DELETE endpoint at /cloudzero/settings - message.warning("Delete functionality is not yet available. Please contact support."); + const handleDeleteClick = () => { + setIsDeleteModalOpen(true); + }; + + const handleDeleteConfirm = () => { + if (!accessToken) return; + + deleteMutation.mutate(undefined, { + onSuccess: () => { + message.success("CloudZero integration deleted successfully"); + setIsDeleteModalOpen(false); + onSettingsUpdated(); + }, + onError: (error) => { + message.error(error?.message || "Failed to delete CloudZero integration"); + }, + }); + }; + + const handleDeleteCancel = () => { + setIsDeleteModalOpen(false); }; return ( @@ -89,19 +110,14 @@ export function CloudZeroIntegrationSettings({ settings, onSettingsUpdated }: Cl - } + onClick={handleDeleteClick} + className="flex items-center gap-2" > - - + Delete + } className="shadow-sm" @@ -187,6 +203,27 @@ export function CloudZeroIntegrationSettings({ settings, onSettingsUpdated }: Cl onCancel={handleEditModalCancel} settings={settings} /> + + ); } From f3b97356fb28bfb25099ed3a58f6eeaa8603ec13 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Fri, 19 Dec 2025 13:00:57 -0800 Subject: [PATCH 07/85] Add lazy loading for GaladrielChatConfig to reduce import memory overhead (#18260) Implements lazy loading pattern for GaladrielChatConfig following the existing approach used for other LLM config classes. This defers the import until the config is actually accessed, reducing memory usage during module initialization. --- litellm/__init__.py | 1 - litellm/_lazy_imports.py | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 9b69beccd79..0463282c089 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1066,7 +1066,6 @@ from .utils import client from .llms.bytez.chat.transformation import BytezChatConfig from .llms.custom_llm import CustomLLM from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig -from .llms.galadriel.chat.transformation import GaladrielChatConfig from .llms.github.chat.transformation import GithubChatConfig from .llms.compactifai.chat.transformation import CompactifAIChatConfig from .llms.empower.chat.transformation import EmpowerChatConfig diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 94b3e4a8da1..14772862686 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -158,6 +158,7 @@ DOTPROMPT_NAMES = ( LLM_CONFIG_NAMES = ( "AmazonConverseConfig", "OpenAILikeChatConfig", + "GaladrielChatConfig", ) # Types that support lazy loading via _lazy_import_types @@ -660,4 +661,12 @@ def _lazy_import_llm_configs(name: str) -> Any: _globals["OpenAILikeChatConfig"] = _OpenAILikeChatConfig return _OpenAILikeChatConfig + if name == "GaladrielChatConfig": + from .llms.galadriel.chat.transformation import ( + GaladrielChatConfig as _GaladrielChatConfig, + ) + + _globals["GaladrielChatConfig"] = _GaladrielChatConfig + return _GaladrielChatConfig + raise AttributeError(f"LLM config lazy import: unknown attribute {name!r}") \ No newline at end of file From 5b1fda02fbbf1e82b1f8d85256dd7b4dd9d7398e Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Fri, 19 Dec 2025 13:44:36 -0800 Subject: [PATCH 08/85] Add infrastructure recommendations to benchmarks documentation (#18264) Added concise PostgreSQL and Redis specifications based on benchmark results and industry standards for API gateway deployments. Includes tiered recommendations for different RPS workloads, configuration best practices, and scaling guidelines. --- docs/my-website/docs/benchmarks.md | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md index 76b61d4c2bd..640212808bd 100644 --- a/docs/my-website/docs/benchmarks.md +++ b/docs/my-website/docs/benchmarks.md @@ -60,6 +60,58 @@ Each machine deploying LiteLLM had the following specs: - Database: PostgreSQL - Redis: Not used +## Infrastructure Recommendations + +Recommended specifications based on benchmark results and industry standards for API gateway deployments. + +### PostgreSQL + +Required for authentication, key management, and usage tracking. + +| Workload | CPU | RAM | Storage | Connections | +|----------|-----|-----|---------|-------------| +| 1-2K RPS | 4-8 cores | 16GB | 200GB SSD (3000+ IOPS) | 100-200 | +| 2-5K RPS | 8 cores | 16-32GB | 500GB SSD (5000+ IOPS) | 200-500 | +| 5K+ RPS | 16+ cores | 32-64GB | 1TB+ SSD (10000+ IOPS) | 500+ | + +**Configuration:** Set `proxy_batch_write_at: 60` to batch writes and reduce DB load. Total connections = pool limit × instances. + +### Redis (Recommended) + +Redis was not used in these benchmarks but provides significant production benefits: 60-80% reduced DB load. + +| Workload | CPU | RAM | +|----------|-----|-----| +| 1-2K RPS | 2-4 cores | 8GB | +| 2-5K RPS | 4 cores | 16GB | +| 5K+ RPS | 8+ cores | 32GB+ | + +**Requirements:** Redis 7.0+, AOF persistence enabled, `allkeys-lru` eviction policy. + +**Configuration:** +```yaml +router_settings: + redis_host: os.environ/REDIS_HOST + redis_port: os.environ/REDIS_PORT + redis_password: os.environ/REDIS_PASSWORD + +litellm_settings: + cache: True + cache_params: + type: redis + host: os.environ/REDIS_HOST + port: os.environ/REDIS_PORT + password: os.environ/REDIS_PASSWORD +``` + +:::tip +Use `redis_host`, `redis_port`, and `redis_password` instead of `redis_url` for ~80 RPS better performance. +::: + +**Scaling:** DB connections scale linearly with instances. Consider PostgreSQL read replicas beyond 5K RPS. + +See [Production Configuration](./proxy/prod) for detailed best practices. + ## Locust Settings - 1000 Users From d9181c188e03e8dc400215190a72693a417a4880 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Fri, 19 Dec 2025 14:19:22 -0800 Subject: [PATCH 09/85] [Refactor] - Lazy load 41 configuration classes (#18267) --- litellm/__init__.py | 83 +++++----- litellm/_lazy_imports.py | 342 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+), 38 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 0463282c089..87b1dec2cd0 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1063,46 +1063,8 @@ from .utils import client # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py # (which imports tiktoken) at import time -from .llms.bytez.chat.transformation import BytezChatConfig from .llms.custom_llm import CustomLLM -from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig -from .llms.github.chat.transformation import GithubChatConfig -from .llms.compactifai.chat.transformation import CompactifAIChatConfig -from .llms.empower.chat.transformation import EmpowerChatConfig -from .llms.huggingface.chat.transformation import HuggingFaceChatConfig -from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig -from .llms.oobabooga.chat.transformation import OobaboogaConfig -from .llms.maritalk import MaritalkConfig -from .llms.openrouter.chat.transformation import OpenrouterConfig -from .llms.datarobot.chat.transformation import DataRobotConfig -from .llms.anthropic.chat.transformation import AnthropicConfig from .llms.anthropic.common_utils import AnthropicModelInfo -from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig -from .llms.groq.stt.transformation import GroqSTTConfig -from .llms.anthropic.completion.transformation import AnthropicTextConfig -from .llms.triton.completion.transformation import TritonConfig -from .llms.triton.completion.transformation import TritonGenerateConfig -from .llms.triton.completion.transformation import TritonInferConfig -from .llms.triton.embedding.transformation import TritonEmbeddingConfig -from .llms.huggingface.rerank.transformation import HuggingFaceRerankConfig -from .llms.databricks.chat.transformation import DatabricksConfig -from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig -from .llms.predibase.chat.transformation import PredibaseConfig -from .llms.replicate.chat.transformation import ReplicateConfig -from .llms.snowflake.chat.transformation import SnowflakeConfig -from .llms.cohere.rerank.transformation import CohereRerankConfig -from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config -from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig -from .llms.infinity.rerank.transformation import InfinityRerankConfig -from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig -from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig -from .llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig -from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig -from .llms.nvidia_nim.rerank.ranking_transformation import NvidiaNimRankingConfig -from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig -from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig -from .llms.voyage.rerank.transformation import VoyageRerankConfig -from .llms.clarifai.chat.transformation import ClarifaiConfig from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config from .llms.meta_llama.chat.transformation import LlamaAPIConfig from .llms.anthropic.experimental_pass_through.messages.transformation import ( @@ -1510,6 +1472,51 @@ if TYPE_CHECKING: from litellm.types.utils import ModelInfo as _ModelInfoType from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.caching.caching import Cache + + # Type stubs for lazy-loaded configs to help mypy + from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig as AmazonConverseConfig + from .llms.openai_like.chat.handler import OpenAILikeChatConfig as OpenAILikeChatConfig + from .llms.galadriel.chat.transformation import GaladrielChatConfig as GaladrielChatConfig + from .llms.github.chat.transformation import GithubChatConfig as GithubChatConfig + from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig as AzureAnthropicConfig + from .llms.bytez.chat.transformation import BytezChatConfig as BytezChatConfig + from .llms.compactifai.chat.transformation import CompactifAIChatConfig as CompactifAIChatConfig + from .llms.empower.chat.transformation import EmpowerChatConfig as EmpowerChatConfig + from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig + from .llms.huggingface.chat.transformation import HuggingFaceChatConfig as HuggingFaceChatConfig + from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig + from .llms.oobabooga.chat.transformation import OobaboogaConfig as OobaboogaConfig + from .llms.maritalk import MaritalkConfig as MaritalkConfig + from .llms.openrouter.chat.transformation import OpenrouterConfig as OpenrouterConfig + from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig + from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig + from .llms.anthropic.completion.transformation import AnthropicTextConfig as AnthropicTextConfig + from .llms.groq.stt.transformation import GroqSTTConfig as GroqSTTConfig + from .llms.triton.completion.transformation import TritonConfig as TritonConfig + from .llms.triton.completion.transformation import TritonGenerateConfig as TritonGenerateConfig + from .llms.triton.completion.transformation import TritonInferConfig as TritonInferConfig + from .llms.triton.embedding.transformation import TritonEmbeddingConfig as TritonEmbeddingConfig + from .llms.huggingface.rerank.transformation import HuggingFaceRerankConfig as HuggingFaceRerankConfig + from .llms.databricks.chat.transformation import DatabricksConfig as DatabricksConfig + from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig as DatabricksEmbeddingConfig + from .llms.predibase.chat.transformation import PredibaseConfig as PredibaseConfig + from .llms.replicate.chat.transformation import ReplicateConfig as ReplicateConfig + from .llms.snowflake.chat.transformation import SnowflakeConfig as SnowflakeConfig + from .llms.cohere.rerank.transformation import CohereRerankConfig as CohereRerankConfig + from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config as CohereRerankV2Config + from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig as AzureAIRerankConfig + from .llms.infinity.rerank.transformation import InfinityRerankConfig as InfinityRerankConfig + from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig as JinaAIRerankConfig + from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig as DeepinfraRerankConfig + from .llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig as HostedVLLMRerankConfig + from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig as NvidiaNimRerankConfig + from .llms.nvidia_nim.rerank.ranking_transformation import NvidiaNimRankingConfig as NvidiaNimRankingConfig + from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig as VertexAIRerankConfig + from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig as FireworksAIRerankConfig + from .llms.voyage.rerank.transformation import VoyageRerankConfig as VoyageRerankConfig + from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig + from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig + from .llms.ai21.chat.transformation import AI21Config as AI21Config from litellm.caching.llm_caching_handler import LLMClientCache from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES from litellm.types.utils import ( diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 14772862686..b25e6830640 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -159,6 +159,44 @@ LLM_CONFIG_NAMES = ( "AmazonConverseConfig", "OpenAILikeChatConfig", "GaladrielChatConfig", + "GithubChatConfig", + "AzureAnthropicConfig", + "BytezChatConfig", + "CompactifAIChatConfig", + "EmpowerChatConfig", + "AiohttpOpenAIChatConfig", + "HuggingFaceChatConfig", + "HuggingFaceEmbeddingConfig", + "OobaboogaConfig", + "MaritalkConfig", + "OpenrouterConfig", + "DataRobotConfig", + "AnthropicConfig", + "AnthropicTextConfig", + "GroqSTTConfig", + "TritonConfig", + "TritonGenerateConfig", + "TritonInferConfig", + "TritonEmbeddingConfig", + "HuggingFaceRerankConfig", + "DatabricksConfig", + "DatabricksEmbeddingConfig", + "PredibaseConfig", + "ReplicateConfig", + "SnowflakeConfig", + "CohereRerankConfig", + "CohereRerankV2Config", + "AzureAIRerankConfig", + "InfinityRerankConfig", + "JinaAIRerankConfig", + "DeepinfraRerankConfig", + "HostedVLLMRerankConfig", + "NvidiaNimRerankConfig", + "NvidiaNimRankingConfig", + "VertexAIRerankConfig", + "FireworksAIRerankConfig", + "VoyageRerankConfig", + "ClarifaiConfig", ) # Types that support lazy loading via _lazy_import_types @@ -669,4 +707,308 @@ def _lazy_import_llm_configs(name: str) -> Any: _globals["GaladrielChatConfig"] = _GaladrielChatConfig return _GaladrielChatConfig + if name == "GithubChatConfig": + from .llms.github.chat.transformation import ( + GithubChatConfig as _GithubChatConfig, + ) + + _globals["GithubChatConfig"] = _GithubChatConfig + return _GithubChatConfig + + if name == "AzureAnthropicConfig": + from .llms.azure_ai.anthropic.transformation import ( + AzureAnthropicConfig as _AzureAnthropicConfig, + ) + + _globals["AzureAnthropicConfig"] = _AzureAnthropicConfig + return _AzureAnthropicConfig + + if name == "BytezChatConfig": + from .llms.bytez.chat.transformation import ( + BytezChatConfig as _BytezChatConfig, + ) + + _globals["BytezChatConfig"] = _BytezChatConfig + return _BytezChatConfig + + if name == "CompactifAIChatConfig": + from .llms.compactifai.chat.transformation import ( + CompactifAIChatConfig as _CompactifAIChatConfig, + ) + + _globals["CompactifAIChatConfig"] = _CompactifAIChatConfig + return _CompactifAIChatConfig + + if name == "EmpowerChatConfig": + from .llms.empower.chat.transformation import ( + EmpowerChatConfig as _EmpowerChatConfig, + ) + + _globals["EmpowerChatConfig"] = _EmpowerChatConfig + return _EmpowerChatConfig + + if name == "AiohttpOpenAIChatConfig": + from .llms.aiohttp_openai.chat.transformation import ( + AiohttpOpenAIChatConfig as _AiohttpOpenAIChatConfig, + ) + + _globals["AiohttpOpenAIChatConfig"] = _AiohttpOpenAIChatConfig + return _AiohttpOpenAIChatConfig + + if name == "HuggingFaceChatConfig": + from .llms.huggingface.chat.transformation import ( + HuggingFaceChatConfig as _HuggingFaceChatConfig, + ) + + _globals["HuggingFaceChatConfig"] = _HuggingFaceChatConfig + return _HuggingFaceChatConfig + + if name == "HuggingFaceEmbeddingConfig": + from .llms.huggingface.embedding.transformation import ( + HuggingFaceEmbeddingConfig as _HuggingFaceEmbeddingConfig, + ) + + _globals["HuggingFaceEmbeddingConfig"] = _HuggingFaceEmbeddingConfig + return _HuggingFaceEmbeddingConfig + + if name == "OobaboogaConfig": + from .llms.oobabooga.chat.transformation import ( + OobaboogaConfig as _OobaboogaConfig, + ) + + _globals["OobaboogaConfig"] = _OobaboogaConfig + return _OobaboogaConfig + + if name == "MaritalkConfig": + from .llms.maritalk import ( + MaritalkConfig as _MaritalkConfig, + ) + + _globals["MaritalkConfig"] = _MaritalkConfig + return _MaritalkConfig + + if name == "OpenrouterConfig": + from .llms.openrouter.chat.transformation import ( + OpenrouterConfig as _OpenrouterConfig, + ) + + _globals["OpenrouterConfig"] = _OpenrouterConfig + return _OpenrouterConfig + + if name == "DataRobotConfig": + from .llms.datarobot.chat.transformation import ( + DataRobotConfig as _DataRobotConfig, + ) + + _globals["DataRobotConfig"] = _DataRobotConfig + return _DataRobotConfig + + if name == "AnthropicConfig": + from .llms.anthropic.chat.transformation import ( + AnthropicConfig as _AnthropicConfig, + ) + + _globals["AnthropicConfig"] = _AnthropicConfig + return _AnthropicConfig + + if name == "AnthropicTextConfig": + from .llms.anthropic.completion.transformation import ( + AnthropicTextConfig as _AnthropicTextConfig, + ) + + _globals["AnthropicTextConfig"] = _AnthropicTextConfig + return _AnthropicTextConfig + + if name == "GroqSTTConfig": + from .llms.groq.stt.transformation import ( + GroqSTTConfig as _GroqSTTConfig, + ) + + _globals["GroqSTTConfig"] = _GroqSTTConfig + return _GroqSTTConfig + + if name == "TritonConfig": + from .llms.triton.completion.transformation import ( + TritonConfig as _TritonConfig, + ) + + _globals["TritonConfig"] = _TritonConfig + return _TritonConfig + + if name == "TritonGenerateConfig": + from .llms.triton.completion.transformation import ( + TritonGenerateConfig as _TritonGenerateConfig, + ) + + _globals["TritonGenerateConfig"] = _TritonGenerateConfig + return _TritonGenerateConfig + + if name == "TritonInferConfig": + from .llms.triton.completion.transformation import ( + TritonInferConfig as _TritonInferConfig, + ) + + _globals["TritonInferConfig"] = _TritonInferConfig + return _TritonInferConfig + + if name == "TritonEmbeddingConfig": + from .llms.triton.embedding.transformation import ( + TritonEmbeddingConfig as _TritonEmbeddingConfig, + ) + + _globals["TritonEmbeddingConfig"] = _TritonEmbeddingConfig + return _TritonEmbeddingConfig + + if name == "HuggingFaceRerankConfig": + from .llms.huggingface.rerank.transformation import ( + HuggingFaceRerankConfig as _HuggingFaceRerankConfig, + ) + + _globals["HuggingFaceRerankConfig"] = _HuggingFaceRerankConfig + return _HuggingFaceRerankConfig + + if name == "DatabricksConfig": + from .llms.databricks.chat.transformation import ( + DatabricksConfig as _DatabricksConfig, + ) + + _globals["DatabricksConfig"] = _DatabricksConfig + return _DatabricksConfig + + if name == "DatabricksEmbeddingConfig": + from .llms.databricks.embed.transformation import ( + DatabricksEmbeddingConfig as _DatabricksEmbeddingConfig, + ) + + _globals["DatabricksEmbeddingConfig"] = _DatabricksEmbeddingConfig + return _DatabricksEmbeddingConfig + + if name == "PredibaseConfig": + from .llms.predibase.chat.transformation import ( + PredibaseConfig as _PredibaseConfig, + ) + + _globals["PredibaseConfig"] = _PredibaseConfig + return _PredibaseConfig + + if name == "ReplicateConfig": + from .llms.replicate.chat.transformation import ( + ReplicateConfig as _ReplicateConfig, + ) + + _globals["ReplicateConfig"] = _ReplicateConfig + return _ReplicateConfig + + if name == "SnowflakeConfig": + from .llms.snowflake.chat.transformation import ( + SnowflakeConfig as _SnowflakeConfig, + ) + + _globals["SnowflakeConfig"] = _SnowflakeConfig + return _SnowflakeConfig + + if name == "CohereRerankConfig": + from .llms.cohere.rerank.transformation import ( + CohereRerankConfig as _CohereRerankConfig, + ) + + _globals["CohereRerankConfig"] = _CohereRerankConfig + return _CohereRerankConfig + + if name == "CohereRerankV2Config": + from .llms.cohere.rerank_v2.transformation import ( + CohereRerankV2Config as _CohereRerankV2Config, + ) + + _globals["CohereRerankV2Config"] = _CohereRerankV2Config + return _CohereRerankV2Config + + if name == "AzureAIRerankConfig": + from .llms.azure_ai.rerank.transformation import ( + AzureAIRerankConfig as _AzureAIRerankConfig, + ) + + _globals["AzureAIRerankConfig"] = _AzureAIRerankConfig + return _AzureAIRerankConfig + + if name == "InfinityRerankConfig": + from .llms.infinity.rerank.transformation import ( + InfinityRerankConfig as _InfinityRerankConfig, + ) + + _globals["InfinityRerankConfig"] = _InfinityRerankConfig + return _InfinityRerankConfig + + if name == "JinaAIRerankConfig": + from .llms.jina_ai.rerank.transformation import ( + JinaAIRerankConfig as _JinaAIRerankConfig, + ) + + _globals["JinaAIRerankConfig"] = _JinaAIRerankConfig + return _JinaAIRerankConfig + + if name == "DeepinfraRerankConfig": + from .llms.deepinfra.rerank.transformation import ( + DeepinfraRerankConfig as _DeepinfraRerankConfig, + ) + + _globals["DeepinfraRerankConfig"] = _DeepinfraRerankConfig + return _DeepinfraRerankConfig + + if name == "HostedVLLMRerankConfig": + from .llms.hosted_vllm.rerank.transformation import ( + HostedVLLMRerankConfig as _HostedVLLMRerankConfig, + ) + + _globals["HostedVLLMRerankConfig"] = _HostedVLLMRerankConfig + return _HostedVLLMRerankConfig + + if name == "NvidiaNimRerankConfig": + from .llms.nvidia_nim.rerank.transformation import ( + NvidiaNimRerankConfig as _NvidiaNimRerankConfig, + ) + + _globals["NvidiaNimRerankConfig"] = _NvidiaNimRerankConfig + return _NvidiaNimRerankConfig + + if name == "NvidiaNimRankingConfig": + from .llms.nvidia_nim.rerank.ranking_transformation import ( + NvidiaNimRankingConfig as _NvidiaNimRankingConfig, + ) + + _globals["NvidiaNimRankingConfig"] = _NvidiaNimRankingConfig + return _NvidiaNimRankingConfig + + if name == "VertexAIRerankConfig": + from .llms.vertex_ai.rerank.transformation import ( + VertexAIRerankConfig as _VertexAIRerankConfig, + ) + + _globals["VertexAIRerankConfig"] = _VertexAIRerankConfig + return _VertexAIRerankConfig + + if name == "FireworksAIRerankConfig": + from .llms.fireworks_ai.rerank.transformation import ( + FireworksAIRerankConfig as _FireworksAIRerankConfig, + ) + + _globals["FireworksAIRerankConfig"] = _FireworksAIRerankConfig + return _FireworksAIRerankConfig + + if name == "VoyageRerankConfig": + from .llms.voyage.rerank.transformation import ( + VoyageRerankConfig as _VoyageRerankConfig, + ) + + _globals["VoyageRerankConfig"] = _VoyageRerankConfig + return _VoyageRerankConfig + + if name == "ClarifaiConfig": + from .llms.clarifai.chat.transformation import ( + ClarifaiConfig as _ClarifaiConfig, + ) + + _globals["ClarifaiConfig"] = _ClarifaiConfig + return _ClarifaiConfig + raise AttributeError(f"LLM config lazy import: unknown attribute {name!r}") \ No newline at end of file From 72f424c719031719b13eba086bb1ad55c3725f4d Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sat, 20 Dec 2025 07:44:40 +0900 Subject: [PATCH 10/85] ensure datadog llm obs ignores dd base url override --- litellm/integrations/datadog/datadog_llm_obs.py | 5 ----- .../datadog/test_datadog_llm_observability.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index b44762d0af8..938ad33f297 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -56,11 +56,6 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): f"https://api.{self.DD_SITE}/api/intake/llm-obs/v1/trace/spans" ) - # testing base url - dd_base_url = os.getenv("DD_BASE_URL") - if dd_base_url: - self.intake_url = f"{dd_base_url}/api/intake/llm-obs/v1/trace/spans" - asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() self.log_queue: List[LLMObsPayload] = [] diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py index 464cb0026e5..c26ea885fa5 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py @@ -293,6 +293,19 @@ class TestDataDogLLMObsLogger: assert logger._get_datadog_span_kind("unknown_call_type") == "llm" assert logger._get_datadog_span_kind(None) == "llm" + def test_dd_base_url_does_not_override_intake_url(self, mock_env_vars): + """Even if DD_BASE_URL is set, intake_url should remain DD_SITE-based""" + with patch.dict(os.environ, {"DD_BASE_URL": "https://example.datadog"}): + with patch( + "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" + ), patch("asyncio.create_task"): + logger = DataDogLLMObsLogger() + + expected_url = ( + f"https://api.{logger.DD_SITE}/api/intake/llm-obs/v1/trace/spans" + ) + assert logger.intake_url == expected_url + @pytest.mark.asyncio async def test_async_log_failure_event(self, mock_env_vars): """Test that async_log_failure_event correctly processes failure payloads according to DD LLM Obs API spec""" From 40b823af87b2c0d8f0b4e7da4b4dce03bb599f70 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 15:13:06 -0800 Subject: [PATCH 11/85] Add Health Check Model for Wildcard in UI --- .../src/components/model_info_view.test.tsx | 65 +++++++++++++++++++ .../src/components/model_info_view.tsx | 55 ++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx index d63d3dd6ebe..21402ad4671 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx @@ -107,6 +107,41 @@ vi.mock("./networking", () => ({ ], }), credentialGetCall: vi.fn().mockResolvedValue({}), + getGuardrailsList: vi.fn().mockResolvedValue({ + guardrails: [{ guardrail_name: "content_filter" }, { guardrail_name: "toxicity_filter" }], + }), + tagListCall: vi.fn().mockResolvedValue({ + test_tag: { + name: "test_tag", + description: "A test tag", + }, + production_tag: { + name: "production_tag", + description: "Production ready models", + }, + }), +})); + +// Mock the useModelsInfo hook since it uses React Query +vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ + useModelsInfo: vi.fn().mockReturnValue({ + data: { + data: [ + { + model_name: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + provider: "bedrock", + litellm_model_name: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + }, + { + model_name: "openai/gpt-4", + provider: "openai", + litellm_model_name: "gpt-4", + }, + ], + }, + isLoading: false, + error: null, + }), })); describe("ModelInfoView", () => { @@ -242,6 +277,36 @@ describe("ModelInfoView", () => { }); }); + it("should render health check model field for wildcard routes", async () => { + const wildcardModelData = { + ...modelData, + litellm_model_name: "openai/gpt-4*", + }; + + const WILDCARD_ADMIN_PROPS = { + ...DEFAULT_ADMIN_PROPS, + modelData: wildcardModelData, + }; + + const { getByText } = render(); + await waitFor(() => { + expect(getByText("Model Settings")).toBeInTheDocument(); + }); + await waitFor(() => { + expect(getByText("Health Check Model")).toBeInTheDocument(); + }); + }); + + it("should not render health check model field for non-wildcard routes", async () => { + const { queryByText } = render(); + await waitFor(() => { + expect(queryByText("Model Settings")).toBeInTheDocument(); + }); + await waitFor(() => { + expect(queryByText("Health Check Model")).not.toBeInTheDocument(); + }); + }); + describe("View Model", () => { it("should render the model info view", async () => { const { getByText } = render(); diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 64f96ac915d..37aa68bcc73 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -37,6 +37,7 @@ import { getProviderLogoAndName } from "./provider_info_helpers"; import NumericalInput from "./shared/numerical_input"; import { Tag } from "./tag_management/types"; import { getDisplayModelName } from "./view_model/model_name_display"; +import { useModelsInfo } from "@/app/(dashboard)/hooks/models/useModels"; interface ModelInfoViewProps { modelId: string; @@ -83,6 +84,8 @@ export default function ModelInfoView({ const isAdmin = userRole === "Admin"; const isAutoRouter = modelData?.litellm_params?.auto_router_config != null; + const { data: modelsInfoData } = useModelsInfo(accessToken, userID, userRole); + console.log("modelsInfoData, ", modelsInfoData); const usingExistingCredential = modelData?.litellm_params?.litellm_credential_name != null && modelData?.litellm_params?.litellm_credential_name != undefined; @@ -226,6 +229,13 @@ export default function ModelInfoView({ access_groups: values.model_access_group, }; } + // Override health_check_model from the form + if (values.health_check_model !== undefined) { + updatedModelInfo = { + ...updatedModelInfo, + health_check_model: values.health_check_model, + }; + } } catch (e) { NotificationsManager.fromBackend("Invalid JSON in Model Info"); return; @@ -342,6 +352,7 @@ export default function ModelInfoView({ onModelUpdate(updatedModel); } }; + const isWildcardModel = modelData.litellm_model_name.includes("*"); return (
@@ -545,6 +556,7 @@ export default function ModelInfoView({ ? localModelData.litellm_params.guardrails : [], tags: Array.isArray(localModelData.litellm_params?.tags) ? localModelData.litellm_params.tags : [], + health_check_model: isWildcardModel ? localModelData.model_info?.health_check_model : null, litellm_extra_params: JSON.stringify(localModelData.litellm_params || {}, null, 2), }} layout="vertical" @@ -868,6 +880,49 @@ export default function ModelInfoView({ )}
+ {isWildcardModel && ( +
+ Health Check Model + {isEditing ? ( + + setDeleteConfirmInput(e.target.value)} - placeholder="Enter key name exactly" - className="w-full px-4 py-3 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500 text-base" - autoFocus - /> -
- - -
- - -
- - - ); - })()} + { + setIsDeleteModalOpen(false); + setDeleteConfirmInput(""); + }} + onOk={handleDelete} + confirmLoading={deleteLoading} + requiredConfirmation={currentKeyData?.key_alias} + /> From 4b652e19d85846a0f7afa86d2a264d359e7204d9 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 20 Dec 2025 17:08:28 -0800 Subject: [PATCH 56/85] =?UTF-8?q?[Fix]=20CI/CD=20-=20security=C2=AD=5Ftest?= =?UTF-8?q?s=20(#18305)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .circleci/config.yml | 12 ++++++++++++ docker/Dockerfile.non_root | 2 +- requirements.txt | 6 +++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b96ca313871..0afacc1d6d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -614,6 +614,12 @@ jobs: - run: name: Install Dependencies command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + python --version + which python + pip install --upgrade typing-extensions>=4.12.0 pip install "pytest==7.3.1" pip install "pytest-asyncio==0.21.1" pip install aiohttp @@ -677,6 +683,9 @@ jobs: - run: name: Run prisma ./docker/entrypoint.sh command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh @@ -685,6 +694,9 @@ jobs: - run: name: Run tests command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv pwd ls python -m pytest tests/proxy_security_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index d8a362680e4..7e9147a124e 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -79,7 +79,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ XDG_CACHE_HOME=/app/.cache \ PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}" -RUN pip install --no-cache-dir prisma==0.11.0 nodejs-bin==18.4.0a4 \ +RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \ && mkdir -p /app/.cache/npm RUN NPM_CONFIG_CACHE=/app/.cache/npm \ diff --git a/requirements.txt b/requirements.txt index 972414a7eba..3bc968c8cb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,14 +13,14 @@ uvloop==0.21.0 # uvicorn dep, gives us much better performance under load boto3==1.36.0 # aws bedrock/sagemaker calls redis==5.2.1 # redis caching prisma==0.11.0 # for db -nodejs-bin==18.4.0a4 ## required by prisma for migrations, prevents runtime download +nodejs-wheel-binaries==24.12.0 ## required by prisma for migrations, prevents runtime download (updated from nodejs-bin for security fixes) mangum==0.17.0 # for aws lambda functions pynacl==1.5.0 # for encrypting keys google-cloud-aiplatform==1.47.0 # for vertex ai calls google-cloud-iam==2.19.1 # for GCP IAM Redis authentication google-genai==1.22.0 anthropic[vertex]==0.54.0 -mcp==1.21.2 ; python_version >= "3.10" # for MCP server +mcp==1.23.0 ; python_version >= "3.10" # for MCP server google-generativeai==0.5.0 # for vertex ai calls async_generator==1.10.0 # for async ollama calls langfuse==2.59.7 # for langfuse self-hosted logging @@ -29,7 +29,7 @@ ddtrace==2.19.0 # for advanced DD tracing / profiling orjson==3.11.2 # fast /embedding responses polars==1.31.0 # for data processing apscheduler==3.10.4 # for resetting budget in background -fastapi-sso==0.16.0 # admin UI, SSO +fastapi-sso==0.19.0 # admin UI, SSO pyjwt[crypto]==2.10.1 ; python_version >= "3.9" python-multipart==0.0.18 # admin UI Pillow==11.0.0 From 7bc98408f0184fb2764b77e1be35e403ddb1e2b1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:21:53 -0800 Subject: [PATCH 57/85] chore: change to reusable buttons and delete modal --- .../LoggingCallbacksTable.tsx | 30 ++------------- .../src/components/budgets/budget_panel.tsx | 16 ++++---- .../src/components/organizations.tsx | 37 ++++++++----------- .../vector_store_management/DeleteModal.tsx | 27 -------------- .../VectorStoreTable.tsx | 17 ++++----- .../vector_store_management/index.tsx | 21 ++++++++--- 6 files changed, 49 insertions(+), 99 deletions(-) delete mode 100644 ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx index 6b79b54ec06..5ad5260c94a 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx @@ -1,10 +1,10 @@ -import { PencilAltIcon, PlayIcon, TrashIcon } from "@heroicons/react/outline"; import { Button, Icon } from "@tremor/react"; import type { TableProps } from "antd"; import { Table, Tooltip } from "antd"; import Title from "antd/es/typography/Title"; import React from "react"; import { AlertingObject } from "./types"; +import TableIconActionButton from "../../../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; type LoggingCallbacksProps = { callbacks: AlertingObject[]; @@ -79,31 +79,9 @@ export const LoggingCallbacksTable: React.FC = ({ align: "right", render: (_: unknown, record: CallbackRow) => (
- - onTest(record)} - /> - - - - onEdit(record)} - /> - - - onDelete(record)} - /> - + onTest(record)} /> + onEdit(record)} /> + onDelete(record)} />
), width: 240, diff --git a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx index 7f581ca6487..2ca05e7161d 100644 --- a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx +++ b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx @@ -3,7 +3,6 @@ * */ -import { PencilAltIcon, TrashIcon } from "@heroicons/react/outline"; import { Button, Card, @@ -28,6 +27,7 @@ import NotificationsManager from "../molecules/notifications_manager"; import { budgetDeleteCall, getBudgetList } from "../networking"; import BudgetModal from "./budget_modal"; import EditBudgetModal from "./edit_budget_modal"; +import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface BudgetSettingsPageProps { accessToken: string | null; @@ -149,16 +149,14 @@ const BudgetPanel: React.FC = ({ accessToken }) => { {value.max_budget ? value.max_budget : "n/a"} {value.tpm_limit ? value.tpm_limit : "n/a"} {value.rpm_limit ? value.rpm_limit : "n/a"} - handleEditCall(value)} /> - handleDeleteClick(value)} /> diff --git a/ui/litellm-dashboard/src/components/organizations.tsx b/ui/litellm-dashboard/src/components/organizations.tsx index 5f7275091e1..32ba210161b 100644 --- a/ui/litellm-dashboard/src/components/organizations.tsx +++ b/ui/litellm-dashboard/src/components/organizations.tsx @@ -23,7 +23,7 @@ import NumericalInput from "./shared/numerical_input"; import { Input } from "antd"; import { Modal, Form, Tooltip, Select as Select2 } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; -import { PencilAltIcon, TrashIcon, RefreshIcon, ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; +import { RefreshIcon, ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; import { TextInput } from "@tremor/react"; import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; import OrganizationInfoView from "./organization/organization_view"; @@ -33,6 +33,7 @@ import MCPServerSelector from "./mcp_server_management/MCPServerSelector"; import { formatNumberWithCommas } from "../utils/dataUtils"; import NotificationsManager from "./molecules/notifications_manager"; import DeleteResourceModal from "./common_components/DeleteResourceModal"; +import TableIconActionButton from "./common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface OrganizationsTableProps { organizations: Organization[]; @@ -375,27 +376,19 @@ const OrganizationsTable: React.FC = ({ {userRole === "Admin" && ( <> - - {" "} - { - setSelectedOrgId(org.organization_id); - setEditOrg(true); - }} - /> - - - {" "} - handleDelete(org.organization_id)} - icon={TrashIcon} - size="sm" - className="cursor-pointer hover:text-red-600" - /> - + { + setSelectedOrgId(org.organization_id); + setEditOrg(true); + }} + /> + handleDelete(org.organization_id)} + /> )} diff --git a/ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx b/ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx deleted file mode 100644 index 34713359d12..00000000000 --- a/ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import React from "react"; -import { Modal } from "antd"; -import { Button as TremorButton } from "@tremor/react"; - -interface DeleteModalProps { - isVisible: boolean; - onCancel: () => void; - onConfirm: () => void; -} - -const DeleteModal: React.FC = ({ isVisible, onCancel, onConfirm }) => { - return ( - -

Are you sure you want to delete this vector store? This action cannot be undone.

-
- - Delete - - - Cancel - -
- - ); -}; - -export default DeleteModal; diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx index b54b5404fde..a5097d8325e 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx @@ -1,6 +1,6 @@ import React from "react"; import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Icon } from "@tremor/react"; -import { TrashIcon, PencilAltIcon, SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; +import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; import { Tooltip } from "antd"; import { ColumnDef, @@ -12,6 +12,7 @@ import { } from "@tanstack/react-table"; import { VectorStore } from "./types"; import { getProviderLogoAndName } from "../provider_info_helpers"; +import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface VectorStoreTableProps { data: VectorStore[]; @@ -104,17 +105,15 @@ const VectorStoreTable: React.FC = ({ data, onView, onEdi const vectorStore = row.original; return (
- onEdit(vectorStore.vector_store_id)} - className="cursor-pointer" /> - onDelete(vectorStore.vector_store_id)} - className="cursor-pointer" />
); diff --git a/ui/litellm-dashboard/src/components/vector_store_management/index.tsx b/ui/litellm-dashboard/src/components/vector_store_management/index.tsx index c8f6ed2d196..6d21e861d4a 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/index.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/index.tsx @@ -5,7 +5,7 @@ import { vectorStoreListCall, vectorStoreDeleteCall, credentialListCall, Credent import { VectorStore } from "./types"; import VectorStoreTable from "./VectorStoreTable"; import VectorStoreForm from "./VectorStoreForm"; -import DeleteModal from "./DeleteModal"; +import DeleteResourceModal from "../common_components/DeleteResourceModal"; import VectorStoreInfoView from "./vector_store_info"; import { isAdminRole } from "@/utils/roles"; import NotificationsManager from "../molecules/notifications_manager"; @@ -25,6 +25,7 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID const [credentials, setCredentials] = useState([]); const [selectedVectorStoreId, setSelectedVectorStoreId] = useState(null); const [editVectorStore, setEditVectorStore] = useState(false); + const [isDeleting, setIsDeleting] = useState(false); const fetchVectorStores = async () => { if (!accessToken) return; @@ -80,6 +81,7 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID const confirmDelete = async () => { if (!accessToken || !vectorStoreToDelete) return; + setIsDeleting(true); try { await vectorStoreDeleteCall(accessToken, vectorStoreToDelete); NotificationsManager.success("Vector store deleted successfully"); @@ -87,9 +89,11 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID } catch (error) { console.error("Error deleting vector store:", error); NotificationsManager.fromBackend("Error deleting vector store: " + error); + } finally { + setIsDeleting(false); + setIsDeleteModalOpen(false); + setVectorStoreToDelete(null); } - setIsDeleteModalOpen(false); - setVectorStoreToDelete(null); }; const handleCreateSuccess = () => { @@ -153,10 +157,15 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID /> {/* Delete Confirmation Modal */} - setIsDeleteModalOpen(false)} - onConfirm={confirmDelete} + onOk={confirmDelete} + confirmLoading={isDeleting} /> From 6e6262b5d2a9b9386aa41abef09e1b9a7a1e1dd1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:25:18 -0800 Subject: [PATCH 58/85] Fixing build --- .../LoggingCallbacks/LoggingCallbacksTable.tsx | 6 +++--- .../src/components/budgets/budget_panel.tsx | 3 +-- .../vector_store_management/VectorStoreTable.tsx | 12 ++++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx index 5ad5260c94a..8f332d0317a 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx @@ -1,10 +1,10 @@ -import { Button, Icon } from "@tremor/react"; +import { Button } from "@tremor/react"; import type { TableProps } from "antd"; -import { Table, Tooltip } from "antd"; +import { Table } from "antd"; import Title from "antd/es/typography/Title"; import React from "react"; -import { AlertingObject } from "./types"; import TableIconActionButton from "../../../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { AlertingObject } from "./types"; type LoggingCallbacksProps = { callbacks: AlertingObject[]; diff --git a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx index 2ca05e7161d..252287191b7 100644 --- a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx +++ b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx @@ -6,7 +6,6 @@ import { Button, Card, - Icon, Tab, TabGroup, Table, @@ -23,11 +22,11 @@ import { import React, { useEffect, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; import NotificationsManager from "../molecules/notifications_manager"; import { budgetDeleteCall, getBudgetList } from "../networking"; import BudgetModal from "./budget_modal"; import EditBudgetModal from "./edit_budget_modal"; -import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface BudgetSettingsPageProps { accessToken: string | null; diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx index a5097d8325e..52462c02e98 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx @@ -1,7 +1,4 @@ -import React from "react"; -import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Icon } from "@tremor/react"; -import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; -import { Tooltip } from "antd"; +import { ChevronDownIcon, ChevronUpIcon, SwitchVerticalIcon } from "@heroicons/react/outline"; import { ColumnDef, flexRender, @@ -10,9 +7,12 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { VectorStore } from "./types"; -import { getProviderLogoAndName } from "../provider_info_helpers"; +import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow } from "@tremor/react"; +import { Tooltip } from "antd"; +import React from "react"; import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { getProviderLogoAndName } from "../provider_info_helpers"; +import { VectorStore } from "./types"; interface VectorStoreTableProps { data: VectorStore[]; From 23477e7621f1d0b77e24f845619ebb52d12b46b4 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 20 Dec 2025 17:32:20 -0800 Subject: [PATCH 59/85] [Fix] CI/CD - test_openai_realtime_direct_call_with_intent (#18308) --- tests/llm_translation/test_openai_realtime.py | 256 ++++++++---------- 1 file changed, 113 insertions(+), 143 deletions(-) diff --git a/tests/llm_translation/test_openai_realtime.py b/tests/llm_translation/test_openai_realtime.py index 91033cf33af..cc40514ec80 100644 --- a/tests/llm_translation/test_openai_realtime.py +++ b/tests/llm_translation/test_openai_realtime.py @@ -25,72 +25,60 @@ async def test_openai_realtime_direct_call_no_intent(): import asyncio import json - # Create a real websocket client that will validate OpenAI responses class RealTimeWebSocketClient: def __init__(self): self.messages_sent = [] self.messages_received = [] self.received_session_created = False self.connection_successful = False + self._receive_called = False async def accept(self): - # Not needed for client-side websocket pass async def send_text(self, message): self.messages_sent.append(message) - # Parse the message to see what we're sending try: - msg_data = json.loads(message) - print(f"Sent to OpenAI: {msg_data.get('type', 'unknown')}") - except json.JSONDecodeError: - pass + if isinstance(message, bytes): + message_str = message.decode('utf-8') + else: + message_str = message + + msg_data = json.loads(message_str) + msg_type = msg_data.get('type', 'unknown') + + if msg_type == "error": + error_info = msg_data.get('error', {}) + error_code = error_info.get('code', 'unknown') + error_message = error_info.get('message', 'unknown') + pytest.fail(f"OpenAI returned error: {error_code} - {error_message}") + + if msg_type == "session.created" and not self.received_session_created: + self.messages_received.append(msg_data) + self.received_session_created = True + self.connection_successful = True + except (json.JSONDecodeError, UnicodeDecodeError) as e: + pytest.fail(f"Failed to parse message: {e}") async def receive_text(self): - # This will be called by the realtime handler when it receives messages from OpenAI - # We'll simulate getting messages for a short time, then close - await asyncio.sleep(0.8) # Give a bit more time for real responses + if not self._receive_called: + self._receive_called = True + max_wait = 60.0 + check_interval = 0.1 + waited = 0.0 + + while waited < max_wait: + if self.connection_successful: + break + await asyncio.sleep(check_interval) + waited += check_interval + + if not self.connection_successful: + await asyncio.sleep(3.0) - # If this is our first call, simulate receiving session.created from OpenAI - if not self.received_session_created: - # This simulates what OpenAI would send on successful connection - response = { - "type": "session.created", - "session": { - "id": "sess_test123", - "object": "realtime.session", - "model": "gpt-4o-realtime-preview-2024-10-01", - "expires_at": 1234567890, - "modalities": ["text", "audio"], - "instructions": "", - "voice": "alloy", - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "input_audio_transcription": None, - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 200 - }, - "tools": [], - "tool_choice": "auto", - "temperature": 0.8, - "max_response_output_tokens": "inf" - } - } - self.messages_received.append(response) - self.received_session_created = True - self.connection_successful = True - print(f"Received from OpenAI: {response['type']}") - return json.dumps(response) - - # After validating we got session.created, close the connection - print("Test validation complete - closing connection") raise websockets.exceptions.ConnectionClosed(None, None) async def close(self, code=1000, reason=""): - # Connection will be closed by the realtime handler pass @property @@ -99,44 +87,29 @@ async def test_openai_realtime_direct_call_no_intent(): websocket_client = RealTimeWebSocketClient() - # Test with no intent parameter - this should NOT produce "Invalid intent" error - # and should receive a valid session.created response try: await litellm._arealtime( model="gpt-4o-realtime-preview-2024-10-01", websocket=websocket_client, api_key=os.environ.get("OPENAI_API_KEY"), - timeout=15 + timeout=60 ) except websockets.exceptions.ConnectionClosed: - # Expected - we close the connection after validation pass - except websockets.exceptions.InvalidStatusCode as e: - # If we get a 4000 status with "invalid_intent", the fix didn't work - if "invalid_intent" in str(e).lower(): - pytest.fail(f"Still getting invalid_intent error: {e}") - else: - # Other connection errors are expected in test environment - pass except Exception as e: - # Make sure we're not getting the "Invalid intent" error - if "invalid_intent" in str(e).lower() or "Invalid intent" in str(e): - pytest.fail(f"Fix failed - still getting invalid intent error: {e}") - # Other exceptions are acceptable for this connection test + if "invalid_intent" in str(e).lower(): + pytest.fail(f"Still getting invalid intent error: {e}") + # Other exceptions (including InvalidStatusCode) are acceptable - # Validate that we successfully connected and received expected response - assert websocket_client.connection_successful, "Failed to establish successful connection to OpenAI" - assert websocket_client.received_session_created, "Did not receive session.created response from OpenAI" - assert len(websocket_client.messages_received) > 0, "No messages received from OpenAI" + assert websocket_client.connection_successful, f"Failed to establish connection. Messages received: {len(websocket_client.messages_sent)}" + assert websocket_client.received_session_created, "Did not receive session.created response" + assert len(websocket_client.messages_received) > 0, "No messages received" - # Validate the structure of the session.created response session_message = websocket_client.messages_received[0] assert session_message["type"] == "session.created", f"Expected session.created, got {session_message.get('type')}" assert "session" in session_message, "session.created response missing session object" assert "id" in session_message["session"], "Session object missing id field" assert "model" in session_message["session"], "Session object missing model field" - - print(f"✅ Successfully validated OpenAI realtime API response structure") @pytest.mark.asyncio @@ -154,72 +127,70 @@ async def test_openai_realtime_direct_call_with_intent(): import asyncio import json - # Create a real websocket client that will validate OpenAI responses class RealTimeWebSocketClient: def __init__(self): self.messages_sent = [] self.messages_received = [] self.received_session_created = False self.connection_successful = False - + self._receive_called = False + self.intent_error_received = None + async def accept(self): - # Not needed for client-side websocket pass - + async def send_text(self, message): self.messages_sent.append(message) - # Parse the message to see what we're sending try: - msg_data = json.loads(message) - print(f"Sent to OpenAI (with intent): {msg_data.get('type', 'unknown')}") - except json.JSONDecodeError: - pass + if isinstance(message, bytes): + message_str = message.decode('utf-8') + else: + message_str = message + + msg_data = json.loads(message_str) + msg_type = msg_data.get('type', 'unknown') + + if msg_type == "error": + error_info = msg_data.get('error', {}) + error_code = error_info.get('code', 'unknown') + error_message = error_info.get('message', 'unknown') + + if error_code == "invalid_intent": + self.intent_error_received = { + 'code': error_code, + 'message': error_message + } + else: + pytest.fail(f"OpenAI returned error: {error_code} - {error_message}") + + if msg_type == "session.created" and not self.received_session_created: + self.messages_received.append(msg_data) + self.received_session_created = True + self.connection_successful = True + except (json.JSONDecodeError, UnicodeDecodeError) as e: + pytest.fail(f"Failed to parse message: {e}") async def receive_text(self): - # This will be called by the realtime handler when it receives messages from OpenAI - await asyncio.sleep(0.8) # Give time for real responses - - # If this is our first call, simulate receiving session.created from OpenAI - if not self.received_session_created: - response = { - "type": "session.created", - "session": { - "id": "sess_intent_test123", - "object": "realtime.session", - "model": "gpt-4o-realtime-preview-2024-10-01", - "expires_at": 1234567890, - "modalities": ["text", "audio"], - "instructions": "", - "voice": "alloy", - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "input_audio_transcription": None, - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 200 - }, - "tools": [], - "tool_choice": "auto", - "temperature": 0.8, - "max_response_output_tokens": "inf" - } - } - self.messages_received.append(response) - self.received_session_created = True - self.connection_successful = True - print(f"Received from OpenAI (with intent): {response['type']}") - return json.dumps(response) - - # After validating we got session.created, close the connection - print("Test validation complete (with intent) - closing connection") + if not self._receive_called: + self._receive_called = True + max_wait = 60.0 + check_interval = 0.1 + waited = 0.0 + + while waited < max_wait: + if self.connection_successful: + break + await asyncio.sleep(check_interval) + waited += check_interval + + if not self.connection_successful: + await asyncio.sleep(3.0) + raise websockets.exceptions.ConnectionClosed(None, None) - + async def close(self, code=1000, reason=""): - # Connection will be closed by the realtime handler pass - + @property def headers(self): return {} @@ -231,41 +202,40 @@ async def test_openai_realtime_direct_call_with_intent(): "intent": "chat" } - # Test with explicit intent parameter try: await litellm._arealtime( model="gpt-4o-realtime-preview-2024-10-01", websocket=websocket_client, api_key=os.environ.get("OPENAI_API_KEY"), query_params=query_params, - timeout=10 + timeout=60 ) except websockets.exceptions.ConnectionClosed: - # Expected - connection closes after brief test - pass - except websockets.exceptions.InvalidStatusCode as e: - # Any connection errors are expected in test environment - # The important thing is we can establish connection without invalid_intent pass except Exception as e: - # Make sure we're not getting unexpected errors - if "invalid_intent" in str(e).lower() or "Invalid intent" in str(e): - pytest.fail(f"Unexpected invalid intent error with explicit intent: {e}") + if "invalid_intent" in str(e).lower(): + pytest.fail(f"Unexpected invalid intent error: {e}") + # Other exceptions (including InvalidStatusCode) are acceptable - # Validate that we successfully connected and received expected response - assert websocket_client.connection_successful, "Failed to establish successful connection to OpenAI (with intent)" - assert websocket_client.received_session_created, "Did not receive session.created response from OpenAI (with intent)" - assert len(websocket_client.messages_received) > 0, "No messages received from OpenAI (with intent)" + if websocket_client.intent_error_received: + websocket_client.connection_successful = True - # Validate the structure of the session.created response - session_message = websocket_client.messages_received[0] - assert session_message["type"] == "session.created", f"Expected session.created, got {session_message.get('type')} (with intent)" - assert "session" in session_message, "session.created response missing session object (with intent)" - assert "id" in session_message["session"], "Session object missing id field (with intent)" - assert "model" in session_message["session"], "Session object missing model field (with intent)" + assert websocket_client.connection_successful, "Failed to establish connection or verify intent parameter pass-through" - print(f"✅ Successfully validated OpenAI realtime API response structure (with intent=chat)") - + if websocket_client.received_session_created: + assert len(websocket_client.messages_received) > 0, "No messages received" + session_message = websocket_client.messages_received[0] + assert session_message["type"] == "session.created", f"Expected session.created, got {session_message.get('type')}" + assert "session" in session_message, "session.created response missing session object" + assert "id" in session_message["session"], "Session object missing id field" + assert "model" in session_message["session"], "Session object missing model field" + elif websocket_client.intent_error_received: + # invalid_intent error confirms intent parameter was passed through + pass + else: + pytest.fail(f"Unexpected test state: connection_successful={websocket_client.connection_successful}, " + f"received_session_created={websocket_client.received_session_created}, " + f"intent_error_received={websocket_client.intent_error_received}") def test_realtime_query_params_construction(): @@ -284,7 +254,7 @@ def test_realtime_query_params_construction(): assert "model" in query_params assert query_params["model"] == model - assert "intent" not in query_params # Should not be present when None + assert "intent" not in query_params # Test case 2: intent is provided (should be included) intent = "chat" @@ -295,4 +265,4 @@ def test_realtime_query_params_construction(): assert "model" in query_params2 assert query_params2["model"] == model assert "intent" in query_params2 - assert query_params2["intent"] == intent \ No newline at end of file + assert query_params2["intent"] == intent From 852bf636984da7e197797603a6123e82fbdbe9ce Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 20 Dec 2025 17:34:08 -0800 Subject: [PATCH 60/85] [Fix] CI/CD - check_code_and_doc_quality (#18309) --- tests/code_coverage_tests/liccheck.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index 328589ac2f8..01d8bc4aa09 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -137,4 +137,5 @@ semantic_router: >=0.1.10 # Unknown license pondpond: >=1.4.1 # Apache 2.0 License fastuuid: >=0.13.0 # BSD-3-Clause license llm-sandbox: >=0.3.31 # MIT License - https://github.com/vndee/llm-sandbox +nodejs-wheel-binaries: >=24.12.0 # MIT license manually verified From 901d145b1a839be53c5149f80f263bd466576d0c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:37:13 -0800 Subject: [PATCH 61/85] Adding UI portion for Agents MD --- AGENTS.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 2c778dc0d71..61afbd035fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,27 @@ LiteLLM is a unified interface for 100+ LLMs that: - Test provider-specific functionality thoroughly - Consider adding load tests for performance-critical changes +### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND) + +1. **Use Common Components as much as possible**: + - These are usually defined in the `common_components` directory + - Use these components as much as possible and avoid building new components unless needed + - Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible + +2. **Testing**: + - The codebase uses **Vitest** and **React Testing Library** + - **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId` + - **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`) + - **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled + - **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present + - **Test names must start with "should"**: All test names should follow the pattern `it("should ...")` + - **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed + - **Structure tests properly**: + - First test should verify the component renders successfully + - Subsequent tests should focus on functionality and user interactions + - Use `waitFor` for async operations that aren't already awaited + - **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation + ### IMPORTANT PATTERNS 1. **Function/Tool Calling**: From f747d12a5f8f9096a4c1cd1e3bd96dee45ff7802 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:51:31 -0800 Subject: [PATCH 62/85] minor styling changes --- .../src/components/cache_dashboard.tsx | 22 +++-- .../organization/organization_view.test.tsx | 23 ++++- .../organization/organization_view.tsx | 87 ++++++++++--------- .../components/team/member_permissions.tsx | 4 +- .../src/components/team/team_info.tsx | 2 +- 5 files changed, 82 insertions(+), 56 deletions(-) diff --git a/ui/litellm-dashboard/src/components/cache_dashboard.tsx b/ui/litellm-dashboard/src/components/cache_dashboard.tsx index 38c0f1a8f41..7b57191a879 100644 --- a/ui/litellm-dashboard/src/components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/cache_dashboard.tsx @@ -1,23 +1,23 @@ -import React, { useState, useEffect } from "react"; import { - Card, BarChart, - Subtitle, - Grid, + Card, Col, DateRangePickerValue, + Grid, + Icon, MultiSelect, MultiSelectItem, - TabPanel, - TabPanels, + Subtitle, + Tab, TabGroup, TabList, - Tab, - Icon, + TabPanel, + TabPanels, Text, } from "@tremor/react"; -import UsageDatePicker from "./shared/usage_date_picker"; +import React, { useEffect, useState } from "react"; import NotificationsManager from "./molecules/notifications_manager"; +import UsageDatePicker from "./shared/usage_date_picker"; import { RefreshIcon } from "@heroicons/react/outline"; import { adminGlobalCacheActivity, cachingHealthCheckCall } from "./networking"; @@ -271,9 +271,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole
Cache Analytics - - 
Cache Health
- + Cache Health Cache Settings
diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx index 0be03169e89..5204efc9411 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { render, waitFor } from "@testing-library/react"; +import { render, screen, waitFor } from "@testing-library/react"; import { vi, test, expect } from "vitest"; import OrganizationInfoView from "./organization_view"; @@ -82,3 +82,24 @@ test("renders organization view after loading data", async () => { expect(findAllByText("Acme Corp")).toBeTruthy(); }); }); + +test("should display empty state when organization has no members", async () => { + const { organizationInfoCall } = await import("../networking"); + (organizationInfoCall as unknown as ReturnType).mockResolvedValueOnce(mockOrg); + + render( + {}} + accessToken="test-token" + is_org_admin={false} + is_proxy_admin={false} + userModels={[]} + editOrg={false} + />, + ); + + await waitFor(() => { + expect(screen.getByText("No members found")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.tsx index 962ec6fa4ea..595987aadf5 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.tsx @@ -324,7 +324,6 @@ const OrganizationInfoView: React.FC = ({ - {/* Budget Panel */}
@@ -340,47 +339,55 @@ const OrganizationInfoView: React.FC = ({ - {orgData.members?.map((member, index) => ( - - - {member.user_id} - - - {member.user_role} - - - ${formatNumberWithCommas(member.spend, 4)} - - - {new Date(member.created_at).toLocaleString()} - - - {canEditOrg && ( - <> - { - setSelectedEditMember({ - role: member.user_role, - user_email: member.user_email, - user_id: member.user_id, - }); - setIsEditMemberModalVisible(true); - }} - /> - { - handleMemberDelete(member); - }} - /> - - )} + {orgData.members && orgData.members.length > 0 ? ( + orgData.members.map((member, index) => ( + + + {member.user_id} + + + {member.user_role} + + + ${formatNumberWithCommas(member.spend, 4)} + + + {new Date(member.created_at).toLocaleString()} + + + {canEditOrg && ( + <> + { + setSelectedEditMember({ + role: member.user_role, + user_email: member.user_email, + user_id: member.user_id, + }); + setIsEditMemberModalVisible(true); + }} + /> + { + handleMemberDelete(member); + }} + /> + + )} + + + )) + ) : ( + + + No members found - ))} + )} diff --git a/ui/litellm-dashboard/src/components/team/member_permissions.tsx b/ui/litellm-dashboard/src/components/team/member_permissions.tsx index 6a7ab541ddf..7eefedb4a2f 100644 --- a/ui/litellm-dashboard/src/components/team/member_permissions.tsx +++ b/ui/litellm-dashboard/src/components/team/member_permissions.tsx @@ -94,9 +94,9 @@ const MemberPermissions: React.FC = ({ teamId, accessTok - +
)} diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index 49a04cce1d1..d2d1c885931 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -508,7 +508,7 @@ const TeamInfoView: React.FC = ({ Back to Teams {info.team_alias} -
+
{info.team_id}
) : (
- {Object.entries(actualSchema.properties).map(([key, prop]) => ( - - {key} {actualSchema.required?.includes(key) && *} - {prop.description && ( - - - - )} - - } - name={key} - rules={[ - { - required: actualSchema.required?.includes(key), + {Object.entries(actualSchema.properties).map(([key, prop]) => { + const initialValue = getInitialValueForField(prop); + const fieldKey = `${tool.name}-${key}`; + return ( + + {key} {actualSchema.required?.includes(key) && *} + {prop.description && ( + + + + )} + + } + name={key} + initialValue={initialValue} + rules={[ + { + required: actualSchema.required?.includes(key), message: `Please enter ${key}`, }, + ...(prop.type === "object" || prop.type === "array" + ? [ + { + validator: (_, value) => { + if ( + (value === undefined || value === null || value === "") && + !actualSchema.required?.includes(key) + ) { + return Promise.resolve(); + } + + try { + const parsed = typeof value === "string" ? JSON.parse(value) : value; + const isValidObject = + prop.type === "object" && + parsed !== null && + typeof parsed === "object" && + !Array.isArray(parsed); + const isValidArray = prop.type === "array" && Array.isArray(parsed); + + if ((prop.type === "object" && isValidObject) || (prop.type === "array" && isValidArray)) { + return Promise.resolve(); + } + + return Promise.reject( + new Error( + prop.type === "object" + ? "Please enter a JSON object" + : "Please enter a JSON array", + ), + ); + } catch (error) { + return Promise.reject(new Error("Invalid JSON")); + } + }, + }, + ] + : []), ]} - className="mb-3" - > - {prop.type === "string" && prop.enum && ( - - )} + className="mb-3" + > + {prop.type === "string" && prop.enum && ( + + )} - {prop.type === "string" && !prop.enum && ( - - )} + {prop.type === "string" && !prop.enum && ( + + )} - {prop.type === "number" && ( - - )} + {(prop.type === "number" || prop.type === "integer") && ( + + )} - {prop.type === "boolean" && ( - - )} - - ))} + {prop.type === "boolean" && ( + + )} + + {(prop.type === "object" || prop.type === "array") && ( +
+