Feat/add delete callback (#11654)

* feat(delete callback): add api and ui for deleting logging callbacks

* hotfix(delete callback): hotfix lint error

* move test case to test_litellm folder

* fix test case to use mock prisma client

litellm/proxy/_types.py

@@ -1463,6 +1463,9 @@ class ConfigFieldDelete(LiteLLMPydanticObjectBase):
     config_type: Literal["general_settings"]
     field_name: str
 
+class CallbackDelete(LiteLLMPydanticObjectBase):
+    callback_name: str
+
 
 class FieldDetail(BaseModel):
     field_name: str

litellm/proxy/proxy_server.py

@@ -8010,6 +8010,95 @@ async def delete_config_general_settings(
     return response
 
 
+@router.post(
+    "/config/callback/delete",
+    tags=["config.yaml"],
+    dependencies=[Depends(user_api_key_auth)],
+    include_in_schema=False,
+)
+async def delete_callback(
+    data: CallbackDelete,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    Delete specific logging callback from configuration.
+    """
+    global prisma_client, proxy_config
+    
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": CommonProxyErrors.db_not_connected_error.value},
+        )
+
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": "{}, your role={}".format(
+                    CommonProxyErrors.not_allowed_access.value,
+                    user_api_key_dict.user_role,
+                )
+            },
+        )
+
+    if store_model_in_db is not True:
+        raise HTTPException(
+            status_code=500,
+            detail={
+                "error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."
+            },
+        )
+
+    try:
+        # Get current configuration
+        config = await proxy_config.get_config()
+        callback_name = data.callback_name.lower()
+        
+        # Check if callback exists in current configuration
+        litellm_settings = config.get("litellm_settings", {})
+        success_callbacks = litellm_settings.get("success_callback", [])
+        
+        if callback_name not in success_callbacks:
+            raise HTTPException(
+                status_code=404,
+                detail={"error": f"Callback '{callback_name}' not found in active configuration"},
+            )
+        
+        # Remove callback from success_callback list
+        success_callbacks.remove(callback_name)
+        config.setdefault("litellm_settings", {})["success_callback"] = success_callbacks
+        
+        # Save the updated configuration
+        await proxy_config.save_config(new_config=config)
+        
+        # Restart the proxy to apply changes
+        await proxy_config.add_deployment(
+            prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
+        )
+
+        return {
+            "message": f"Successfully deleted callback: {callback_name}",
+            "removed_callback": callback_name,
+            "remaining_callbacks": success_callbacks,
+            "deleted_at": datetime.now().isoformat(),
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        verbose_proxy_logger.error(
+            f"litellm.proxy.proxy_server.delete_callback(): Exception occurred - {str(e)}"
+        )
+        verbose_proxy_logger.debug(traceback.format_exc())
+        raise ProxyException(
+            message="Error deleting callback: " + str(e),
+            type=ProxyErrorTypes.internal_server_error,
+            param="callback_name",
+            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
+
+
 @router.get(
     "/get/config/callbacks",
     tags=["config.yaml"],

tests/test_litellm/proxy/management_endpoints/test_delete_callbacks_endpoint.py

@@ -0,0 +1,176 @@
+import os
+import sys
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from fastapi.testclient import TestClient
+
+sys.path.insert(
+    0, os.path.abspath("../../../..")
+)
+
+from litellm.proxy._types import CallbackDelete, ConfigYAML, LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy.proxy_server import app
+
+client = TestClient(app)
+
+
+class MockPrismaClient:
+    def __init__(self):
+        self.db = MagicMock()
+        self.config_data = {
+            "litellm_settings": {"success_callback": ["langfuse"]},
+            "environment_variables": {
+                "LANGFUSE_PUBLIC_KEY": "any-public-key",
+                "LANGFUSE_SECRET_KEY": "any-secret-key", 
+                "LANGFUSE_HOST": "https://exampleopenaiendpoint-production-c715.up.railway.app",
+            },
+        }
+        
+        # Mock the config update/upsert
+        self.db.litellm_config.upsert = AsyncMock()
+        
+        # Mock config retrieval for get_config/callbacks
+        self.db.litellm_config.find_first = AsyncMock(
+            side_effect=self._mock_find_first
+        )
+        
+        # Mock for get_generic_data
+        self.get_generic_data = AsyncMock(side_effect=self._mock_get_generic_data)
+        
+        # Mock insert_data method (required by delete_callback endpoint)
+        self.insert_data = AsyncMock(return_value=MagicMock())
+        
+        # Mock jsonify_object method (required by config endpoints)
+        self.jsonify_object = lambda obj: obj
+
+    async def _mock_find_first(self, where=None):
+        """Mock find_first to return config data based on param_name"""
+        if where and "param_name" in where:
+            param_name = where["param_name"]
+            if param_name == "litellm_settings":
+                return MagicMock(
+                    param_name="litellm_settings",
+                    param_value=self.config_data["litellm_settings"]
+                )
+            elif param_name == "environment_variables":
+                return MagicMock(
+                    param_name="environment_variables", 
+                    param_value=self.config_data["environment_variables"]
+                )
+        return None
+
+    async def _mock_get_generic_data(self, key=None, value=None, table_name=None):
+        """Mock get_generic_data for _update_config_from_db"""
+        if key == "param_name" and table_name == "config":
+            if value == "litellm_settings":
+                return MagicMock(
+                    param_name="litellm_settings",
+                    param_value=self.config_data["litellm_settings"]
+                )
+            elif value == "environment_variables":
+                return MagicMock(
+                    param_name="environment_variables",
+                    param_value=self.config_data["environment_variables"]
+                )
+            elif value in ["general_settings", "router_settings"]:
+                return None
+        return None
+
+    def remove_callback_from_config(self, callback_name):
+        """Remove callback from the mock config"""
+        if "success_callback" in self.config_data["litellm_settings"]:
+            callbacks = self.config_data["litellm_settings"]["success_callback"]
+            if callback_name in callbacks:
+                callbacks.remove(callback_name)
+
+
+@pytest.fixture
+def mock_auth():
+    """Mock admin user authentication"""
+    return UserAPIKeyAuth(
+        user_id="test_admin",
+        user_role=LitellmUserRoles.PROXY_ADMIN,
+        api_key="sk-1234"
+    )
+
+
+@pytest.fixture
+def mock_prisma():
+    """Mock prisma client"""
+    return MockPrismaClient()
+
+
+def mock_encrypt_value_helper(value):
+    """Mock encryption - just return the value as-is for testing"""
+    return value
+
+def mock_decrypt_value_helper(value):
+    """Mock decryption - just return the value as-is for testing"""
+    return value
+
+
+@pytest.mark.asyncio
+async def test_delete_callbacks_in_db(mock_prisma, mock_auth):
+    
+    with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), \
+         patch("litellm.proxy.proxy_server.store_model_in_db", True), \
+         patch("litellm.proxy.proxy_server.encrypt_value_helper", side_effect=mock_encrypt_value_helper), \
+         patch("litellm.proxy.proxy_server.decrypt_value_helper", side_effect=mock_decrypt_value_helper):
+        
+        # Override auth dependency
+        app.dependency_overrides[
+            lambda: __import__("litellm.proxy.proxy_server", fromlist=["user_api_key_auth"]).user_api_key_auth
+        ] = lambda: mock_auth
+        
+        # Add langfuse callback to DB via /config/update
+        config_data = {
+            "litellm_settings": {"success_callback": ["langfuse"]},
+            "environment_variables": {
+                "LANGFUSE_PUBLIC_KEY": "any-public-key",
+                "LANGFUSE_SECRET_KEY": "any-secret-key",
+                "LANGFUSE_HOST": "https://exampleopenaiendpoint-production-c715.up.railway.app",
+            },
+        }
+        
+        config_response = client.post(
+            "/config/update", 
+            json=config_data,
+            headers={"Authorization": "Bearer sk-1234"}
+        )
+        assert config_response.status_code == 200
+        
+        # Delete the langfuse callback
+        delete_data = {"callback_name": "langfuse"}
+        delete_response = client.post(
+            "/config/callback/delete",
+            json=delete_data, 
+            headers={"Authorization": "Bearer sk-1234"}
+        )
+        
+        assert delete_response.status_code == 200
+        delete_result = delete_response.json()
+        
+        # Verify delete response
+        assert "message" in delete_result
+        assert "langfuse" in delete_result.get("removed_callback", "")
+        assert "langfuse" not in delete_result.get("remaining_callbacks", [])
+        
+        # Update mock to reflect deletion for get_config test
+        mock_prisma.remove_callback_from_config("langfuse")
+        
+        # Get config and verify callback is deleted
+        config_response = client.get(
+            "/get/config/callbacks",
+            headers={"Authorization": "Bearer sk-1234"}
+        )
+        
+        assert config_response.status_code == 200
+        config_data = config_response.json()
+        
+        # Verify callback is removed from the config
+        callback_names = [callback["name"] for callback in config_data.get("callbacks", [])]
+        assert "langfuse" not in callback_names
+        
+        # Clean up
+        app.dependency_overrides.clear() 

ui/litellm-dashboard/src/components/settings.tsx

@@ -28,7 +28,8 @@ import {
 } from "@tremor/react";
 
 import {
-  PencilAltIcon
+  PencilAltIcon,
+  TrashIcon
 } from "@heroicons/react/outline";
 
 import { Modal, Typography, Form, Input, Select, Button as Button2, message } from "antd";
@@ -40,6 +41,7 @@ import {
   getCallbacksCall,
   setCallbacksCall,
   serviceHealthCheck,
+  deleteCallback
 } from "./networking";
 import AlertingSettings from "./alerting/alerting_settings";
 import FormItem from "antd/es/form/FormItem";
@@ -132,6 +134,8 @@ const Settings: React.FC<SettingsPageProps> = ({
 
   const [showEditCallback, setShowEditCallback] = useState(false);
   const [selectedEditCallback, setSelectedEditCallback] = useState<any | null>(null);
+  const [showDeleteConfirmModal, setShowDeleteConfirmModal] = useState(false);
+  const [callbackToDelete, setCallbackToDelete] = useState<string | null>(null);
 
   const handleSwitchChange = (alertName: string) => {
     if (activeAlerts.includes(alertName)) {
@@ -447,6 +451,33 @@ const Settings: React.FC<SettingsPageProps> = ({
   const handleCallbackChange = (value: string) => {
     setSelectedCallback(value);
   };
+  const handleDeleteCallback = (callbackName: string) => {
+    setCallbackToDelete(callbackName);
+    setShowDeleteConfirmModal(true);
+  };
+
+  const confirmDeleteCallback = async () => {
+    if (!callbackToDelete || !accessToken) {
+      return;
+    }
+
+    try {
+      await deleteCallback(accessToken, callbackToDelete);
+      message.success(`Callback ${callbackToDelete} deleted successfully`);
+
+      // Refresh the callbacks list
+      if (userID && userRole) {
+        const data = await getCallbacksCall(accessToken, userID, userRole);
+        setCallbacks(data.callbacks);
+      }
+
+      setShowDeleteConfirmModal(false);
+      setCallbackToDelete(null);
+    } catch (error) {
+      console.error("Failed to delete callback:", error);
+      message.error(`Failed to delete callback: ${error}`);
+    }
+  };
 
   if (!accessToken) {
     return null;
@@ -498,6 +529,14 @@ const Settings: React.FC<SettingsPageProps> = ({
                                   setShowEditCallback(true);
                                 }}
                               />
+                              <Icon
+                                icon={TrashIcon}
+                                size="sm"
+                                onClick={() =>
+                                  handleDeleteCallback(callback.name)
+                                }
+                                className="text-red-500 hover:text-red-700 cursor-pointer"
+                              />
                               <Button
                                 onClick={() =>
                                   serviceHealthCheck(accessToken, callback.name)
@@ -730,6 +769,24 @@ const Settings: React.FC<SettingsPageProps> = ({
         
 
     </Modal>
+
+    <Modal
+      title="Confirm Delete"
+      visible={showDeleteConfirmModal}
+      onOk={confirmDeleteCallback}
+      onCancel={() => {
+        setShowDeleteConfirmModal(false);
+        setCallbackToDelete(null);
+      }}
+      okText="Delete"
+      cancelText="Cancel"
+      okButtonProps={{ danger: true }}
+    >
+      <p>
+        Are you sure you want to delete the callback "{callbackToDelete}"?
+        This action cannot be undone.
+      </p>
+    </Modal>
     </div>
     
   );