diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 13bbf97beb3..f4a7b30126f 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -782,6 +782,12 @@ async def add_litellm_data_to_request( # noqa: PLR0915 for k, v in callback_settings_obj.callback_vars.items(): data[k] = v + # Add disabled callbacks from key metadata + if user_api_key_dict.metadata and "litellm_disabled_callbacks" in user_api_key_dict.metadata: + disabled_callbacks = user_api_key_dict.metadata["litellm_disabled_callbacks"] + if disabled_callbacks and isinstance(disabled_callbacks, list): + data["litellm_disabled_callbacks"] = disabled_callbacks + # Guardrails move_guardrails_to_metadata( data=data, diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 626d82e95cc..33322150407 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -220,6 +220,279 @@ async def test_add_litellm_data_to_request_audio_transcription_multipart(): ] +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_disabled_callbacks(): + """ + Test that litellm_disabled_callbacks from key metadata is properly added to the request data. + """ + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + # Setup mock request + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + # Setup user API key with disabled callbacks in metadata + user_api_key_dict = UserAPIKeyAuth( + api_key="test_api_key", + user_id="test_user_id", + org_id="test_org_id", + metadata={ + "litellm_disabled_callbacks": ["langfuse", "langsmith", "datadog"] + } + ) + + # Setup request data + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + # Setup proxy config + proxy_config = MagicMock() + + # Call add_litellm_data_to_request + result = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config + ) + + # Verify that litellm_disabled_callbacks was added to the request data + assert "litellm_disabled_callbacks" in result + assert result["litellm_disabled_callbacks"] == ["langfuse", "langsmith", "datadog"] + + # Verify that other data is still present + assert "model" in result + assert result["model"] == "gpt-3.5-turbo" + assert "messages" in result + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_disabled_callbacks_empty(): + """ + Test that litellm_disabled_callbacks is not added when it's empty. + """ + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + # Setup mock request + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + # Setup user API key with empty disabled callbacks + user_api_key_dict = UserAPIKeyAuth( + api_key="test_api_key", + user_id="test_user_id", + org_id="test_org_id", + metadata={ + "litellm_disabled_callbacks": [] + } + ) + + # Setup request data + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + # Setup proxy config + proxy_config = MagicMock() + + # Call add_litellm_data_to_request + result = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config + ) + + # Verify that litellm_disabled_callbacks is not added when empty + assert "litellm_disabled_callbacks" not in result + + # Verify that other data is still present + assert "model" in result + assert result["model"] == "gpt-3.5-turbo" + assert "messages" in result + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_disabled_callbacks_not_present(): + """ + Test that litellm_disabled_callbacks is not added when it's not present in metadata. + """ + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + # Setup mock request + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + # Setup user API key without disabled callbacks in metadata + user_api_key_dict = UserAPIKeyAuth( + api_key="test_api_key", + user_id="test_user_id", + org_id="test_org_id", + metadata={} # No litellm_disabled_callbacks + ) + + # Setup request data + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + # Setup proxy config + proxy_config = MagicMock() + + # Call add_litellm_data_to_request + result = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config + ) + + # Verify that litellm_disabled_callbacks is not added when not present + assert "litellm_disabled_callbacks" not in result + + # Verify that other data is still present + assert "model" in result + assert result["model"] == "gpt-3.5-turbo" + assert "messages" in result + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_disabled_callbacks_invalid_type(): + """ + Test that litellm_disabled_callbacks is not added when it's not a list. + """ + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + # Setup mock request + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + # Setup user API key with invalid disabled callbacks type + user_api_key_dict = UserAPIKeyAuth( + api_key="test_api_key", + user_id="test_user_id", + org_id="test_org_id", + metadata={ + "litellm_disabled_callbacks": "not_a_list" # Should be a list + } + ) + + # Setup request data + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + # Setup proxy config + proxy_config = MagicMock() + + # Call add_litellm_data_to_request + result = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config + ) + + # Verify that litellm_disabled_callbacks is not added when invalid type + assert "litellm_disabled_callbacks" not in result + + # Verify that other data is still present + assert "model" in result + assert result["model"] == "gpt-3.5-turbo" + assert "messages" in result + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_disabled_callbacks_with_logging_settings(): + """ + Test that litellm_disabled_callbacks works correctly alongside logging settings. + """ + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + # Setup mock request + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + # Setup user API key with both logging settings and disabled callbacks + user_api_key_dict = UserAPIKeyAuth( + api_key="test_api_key", + user_id="test_user_id", + org_id="test_org_id", + metadata={ + "logging": [ + {"callback_name": "langfuse", "callback_type": "success", "callback_vars": {}} + ], + "litellm_disabled_callbacks": ["langsmith", "datadog"] + } + ) + + # Setup request data + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + # Setup proxy config + proxy_config = MagicMock() + + # Call add_litellm_data_to_request + result = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config + ) + + # Verify that both logging settings and disabled callbacks are handled correctly + assert "litellm_disabled_callbacks" in result + assert result["litellm_disabled_callbacks"] == ["langsmith", "datadog"] + + # Verify that other data is still present + assert "model" in result + assert result["model"] == "gpt-3.5-turbo" + assert "messages" in result + + def test_key_dynamic_logging_settings(): """ Test KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings method with arize and langfuse callbacks diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index d89b49324ed..095ae54ea77 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -1,7 +1,7 @@ export enum Callbacks { Braintrust = "Braintrust", CustomCallbackAPI = "Custom Callback API", - Datadog = "Datagog", + Datadog = "Datadog", Langfuse = "Langfuse", LangSmith = "LangSmith", Lago = "Lago", @@ -24,7 +24,7 @@ export const callback_map: Record = { Arize: "arize", } -const asset_logos_folder = '/ui/assets/logos/'; +const asset_logos_folder = '/assets/logos/'; interface CallbackInfo { logo: string; @@ -66,7 +66,7 @@ export const callbackInfo: Record = { }, [Callbacks.CustomCallbackAPI]: { logo: `${asset_logos_folder}custom.svg`, - supports_key_team_logging: false, + supports_key_team_logging: true, dynamic_params: {} }, [Callbacks.Datadog]: { diff --git a/ui/litellm-dashboard/src/components/common_components/PremiumLoggingSettings.tsx b/ui/litellm-dashboard/src/components/common_components/PremiumLoggingSettings.tsx index b9dd9cebf2d..238dca25cfc 100644 --- a/ui/litellm-dashboard/src/components/common_components/PremiumLoggingSettings.tsx +++ b/ui/litellm-dashboard/src/components/common_components/PremiumLoggingSettings.tsx @@ -6,12 +6,16 @@ interface PremiumLoggingSettingsProps { value: any[]; onChange: (settings: any[]) => void; premiumUser?: boolean; + disabledCallbacks?: string[]; + onDisabledCallbacksChange?: (disabledCallbacks: string[]) => void; } export function PremiumLoggingSettings({ value, onChange, - premiumUser = false + premiumUser = false, + disabledCallbacks = [], + onDisabledCallbacksChange }: PremiumLoggingSettingsProps) { if (!premiumUser) { return ( @@ -37,6 +41,8 @@ export function PremiumLoggingSettings({ ); } diff --git a/ui/litellm-dashboard/src/components/create_key_button.tsx b/ui/litellm-dashboard/src/components/create_key_button.tsx index d95a828a8a8..17e26c8e2ca 100644 --- a/ui/litellm-dashboard/src/components/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/create_key_button.tsx @@ -181,11 +181,13 @@ const CreateKey: React.FC = ({ const [userSearchLoading, setUserSearchLoading] = useState(false); const [mcpAccessGroups, setMcpAccessGroups] = useState([]); const [mcpAccessGroupsLoaded, setMcpAccessGroupsLoaded] = useState(false); + const [disabledCallbacks, setDisabledCallbacks] = useState([]); const handleOk = () => { setIsModalVisible(false); form.resetFields(); setLoggingSettings([]); + setDisabledCallbacks([]); }; const handleCancel = () => { @@ -194,6 +196,7 @@ const CreateKey: React.FC = ({ setSelectedCreateKeyTeam(null); form.resetFields(); setLoggingSettings([]); + setDisabledCallbacks([]); }; useEffect(() => { @@ -306,6 +309,14 @@ const CreateKey: React.FC = ({ }; } + // Add disabled callbacks to the metadata + if (disabledCallbacks.length > 0) { + metadata = { + ...metadata, + litellm_disabled_callbacks: disabledCallbacks + }; + } + // Update the formValues with the final metadata formValues.metadata = JSON.stringify(metadata); @@ -866,6 +877,8 @@ const CreateKey: React.FC = ({ value={loggingSettings} onChange={setLoggingSettings} premiumUser={premiumUser} + disabledCallbacks={disabledCallbacks} + onDisabledCallbacksChange={setDisabledCallbacks} /> diff --git a/ui/litellm-dashboard/src/components/key_edit_view.tsx b/ui/litellm-dashboard/src/components/key_edit_view.tsx index fe6bc87710d..33405d8034e 100644 --- a/ui/litellm-dashboard/src/components/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/key_edit_view.tsx @@ -57,6 +57,7 @@ export function KeyEditView({ const [availableModels, setAvailableModels] = useState([]); const [mcpAccessGroups, setMcpAccessGroups] = useState([]); const [mcpAccessGroupsLoaded, setMcpAccessGroupsLoaded] = useState(false); + const [disabledCallbacks, setDisabledCallbacks] = useState(Array.isArray(keyData.metadata?.litellm_disabled_callbacks) ? keyData.metadata.litellm_disabled_callbacks : []); const fetchMcpAccessGroups = async () => { if (!accessToken) return; @@ -99,6 +100,11 @@ export function KeyEditView({ fetchModels(); }, [userID, userRole, accessToken, team, keyData.team_id]); + // Sync disabled callbacks with form when component mounts + useEffect(() => { + form.setFieldValue('disabled_callbacks', disabledCallbacks); + }, [form, disabledCallbacks]); + // Convert API budget duration to form format const getBudgetDuration = (duration: string | null) => { if (!duration) return null; @@ -121,7 +127,8 @@ export function KeyEditView({ servers: keyData.object_permission?.mcp_servers || [], accessGroups: keyData.object_permission?.mcp_access_groups || [] }, - logging_settings: extractLoggingSettings(keyData.metadata) + logging_settings: extractLoggingSettings(keyData.metadata), + disabled_callbacks: Array.isArray(keyData.metadata?.litellm_disabled_callbacks) ? keyData.metadata.litellm_disabled_callbacks : [] }; return ( @@ -229,6 +236,11 @@ export function KeyEditView({ form.setFieldValue('logging_settings', values)} + disabledCallbacks={disabledCallbacks} + onDisabledCallbacksChange={(values) => { + setDisabledCallbacks(values); + form.setFieldValue('disabled_callbacks', values); + }} /> @@ -243,6 +255,11 @@ export function KeyEditView({ + {/* Hidden form field for disabled callbacks */} + +
diff --git a/ui/litellm-dashboard/src/components/key_info_view.tsx b/ui/litellm-dashboard/src/components/key_info_view.tsx index fa0e1818d1b..d525d98e225 100644 --- a/ui/litellm-dashboard/src/components/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/key_info_view.tsx @@ -98,8 +98,15 @@ export default function KeyInfoView({ keyId, onClose, keyData, accessToken, user const parsedMetadata = JSON.parse(formValues.metadata); formValues.metadata = { ...parsedMetadata, - ...(formValues.guardrails?.length > 0 ? { guardrails: formValues.guardrails } : {}), - ...(formValues.logging_settings ? { logging: formValues.logging_settings } : {}) + ...(formValues.guardrails?.length > 0 + ? { guardrails: formValues.guardrails } + : {}), + ...(formValues.logging_settings + ? { logging: formValues.logging_settings } + : {}), + ...(formValues.disabled_callbacks?.length > 0 + ? { litellm_disabled_callbacks: formValues.disabled_callbacks } + : {}), }; } catch (error) { console.error("Error parsing metadata JSON:", error); @@ -109,8 +116,15 @@ export default function KeyInfoView({ keyId, onClose, keyData, accessToken, user } else { formValues.metadata = { ...(formValues.metadata || {}), - ...(formValues.guardrails?.length > 0 ? { guardrails: formValues.guardrails } : {}), - ...(formValues.logging_settings ? { logging: formValues.logging_settings } : {}) + ...(formValues.guardrails?.length > 0 + ? { guardrails: formValues.guardrails } + : {}), + ...(formValues.logging_settings + ? { logging: formValues.logging_settings } + : {}), + ...(formValues.disabled_callbacks?.length > 0 + ? { litellm_disabled_callbacks: formValues.disabled_callbacks } + : {}), }; } @@ -321,7 +335,8 @@ export default function KeyInfoView({ keyId, onClose, keyData, accessToken, user @@ -442,7 +457,10 @@ export default function KeyInfoView({ keyId, onClose, keyData, accessToken, user /> diff --git a/ui/litellm-dashboard/src/components/logging_settings_view.tsx b/ui/litellm-dashboard/src/components/logging_settings_view.tsx index 48344972c88..43c63121b36 100644 --- a/ui/litellm-dashboard/src/components/logging_settings_view.tsx +++ b/ui/litellm-dashboard/src/components/logging_settings_view.tsx @@ -1,6 +1,6 @@ import React from "react"; import { Text, Badge } from "@tremor/react"; -import { CogIcon } from "@heroicons/react/outline"; +import { CogIcon, BanIcon } from "@heroicons/react/outline"; import { callbackInfo, callback_map } from "./callback_info_helpers"; interface LoggingConfig { @@ -11,12 +11,14 @@ interface LoggingConfig { interface LoggingSettingsViewProps { loggingConfigs?: LoggingConfig[]; + disabledCallbacks?: string[]; variant?: "card" | "inline"; className?: string; } export function LoggingSettingsView({ loggingConfigs = [], + disabledCallbacks = [], variant = "card", className = "" }: LoggingSettingsViewProps) { @@ -55,59 +57,117 @@ export function LoggingSettingsView({ }; const content = ( -
-
- - Logging Integrations - - {loggingConfigs.length} - -
- - {loggingConfigs.length > 0 ? ( -
- {loggingConfigs.map((config, index) => { - const displayName = getLoggingDisplayName(config.callback_name); - const logoUrl = callbackInfo[displayName]?.logo; - - return ( -
-
- {logoUrl ? ( - {displayName} - ) : ( - - )} -
- {displayName} - - {Object.keys(config.callback_vars).length} parameters configured - -
-
- + {/* Logging Integrations Section */} +
+
+ + Logging Integrations + + {loggingConfigs.length} + +
+ + {loggingConfigs.length > 0 ? ( +
+ {loggingConfigs.map((config, index) => { + const displayName = getLoggingDisplayName(config.callback_name); + const logoUrl = callbackInfo[displayName]?.logo; + + return ( +
- {getEventTypeLabel(config.callback_type)} - -
- ); - })} +
+ {logoUrl ? ( + {displayName} + ) : ( + + )} +
+ {displayName} + + {Object.keys(config.callback_vars).length} parameters configured + +
+
+ + {getEventTypeLabel(config.callback_type)} + +
+ ); + })} +
+ ) : ( +
+ + No logging integrations configured +
+ )} +
+ + {/* Disabled Callbacks Section */} +
+
+ + Disabled Callbacks + + {disabledCallbacks.length} +
- ) : ( -
- - No logging integrations configured -
- )} + + {disabledCallbacks.length > 0 ? ( +
+ {disabledCallbacks.map((callbackName, index) => { + const logoUrl = callbackInfo[callbackName]?.logo; + + return ( +
+
+ {logoUrl ? ( + {callbackName} + ) : ( + + )} +
+ {callbackName} + + Disabled for this key + +
+
+ + Disabled + +
+ ); + })} +
+ ) : ( +
+ + No callbacks disabled +
+ )} +
); @@ -118,7 +178,7 @@ export function LoggingSettingsView({
Logging Settings - Active logging integrations for this team + Active logging integrations and disabled callbacks for this key
diff --git a/ui/litellm-dashboard/src/components/team/EditLoggingSettings.tsx b/ui/litellm-dashboard/src/components/team/EditLoggingSettings.tsx index 13bc8595146..7ac265a9bc3 100644 --- a/ui/litellm-dashboard/src/components/team/EditLoggingSettings.tsx +++ b/ui/litellm-dashboard/src/components/team/EditLoggingSettings.tsx @@ -4,14 +4,28 @@ import LoggingSettings from './LoggingSettings'; interface EditLoggingSettingsProps { value: any[]; onChange: (value: any[]) => void; + disabledCallbacks?: string[]; + onDisabledCallbacksChange?: (disabledCallbacks: string[]) => void; } /** * Wrapper component around LoggingSettings used for editing * a team's logging integrations. */ -const EditLoggingSettings: React.FC = ({ value, onChange }) => { - return ; +const EditLoggingSettings: React.FC = ({ + value, + onChange, + disabledCallbacks = [], + onDisabledCallbacksChange +}) => { + return ( + + ); }; export default EditLoggingSettings; diff --git a/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx b/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx index 32e45b9ee2a..8aeb60ee21d 100644 --- a/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx +++ b/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx @@ -4,7 +4,7 @@ import React, { useState } from 'react'; import { Form, Select, Space, Tooltip, Divider } from 'antd'; import { InfoCircleOutlined } from '@ant-design/icons'; import { Button, Card, TextInput } from '@tremor/react'; -import { PlusIcon, TrashIcon, CogIcon } from '@heroicons/react/outline'; +import { PlusIcon, TrashIcon, CogIcon, BanIcon } from '@heroicons/react/outline'; import { callbackInfo, Callbacks, callback_map } from '../callback_info_helpers'; const { Option } = Select; @@ -18,18 +18,32 @@ interface LoggingConfig { interface LoggingSettingsProps { value?: LoggingConfig[]; onChange?: (value: LoggingConfig[]) => void; + disabledCallbacks?: string[]; + onDisabledCallbacksChange?: (disabledCallbacks: string[]) => void; } -const LoggingSettings: React.FC = ({ value = [], onChange }) => { +const LoggingSettings: React.FC = ({ + value = [], + onChange, + disabledCallbacks = [], + onDisabledCallbacksChange +}) => { // Get callbacks that support team and key logging const supportedCallbacks = Object.entries(callbackInfo) .filter(([_, info]) => info.supports_key_team_logging) .map(([name, _]) => name); + // Get all available callbacks for disabled selection + const allCallbacks = Object.keys(callbackInfo); + const handleChange = (newValue: LoggingConfig[]) => { onChange?.(newValue); }; + const handleDisabledCallbacksChange = (newDisabledCallbacks: string[]) => { + onDisabledCallbacksChange?.(newDisabledCallbacks); + }; + const addLoggingConfig = () => { const newConfig: LoggingConfig = { callback_name: '', @@ -126,6 +140,53 @@ const LoggingSettings: React.FC = ({ value = [], onChange return (
+ {/* Disabled Callbacks Section */} +
+
+ + Disabled Callbacks + + + +
+ +
+ + +
+ Select callbacks that should be disabled for this key. These callbacks will not receive any logging data. +
+
+
+ + + + {/* Logging Integrations Section */}
diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index 22b3a1437ef..402f0b26e98 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -502,6 +502,7 @@ const TeamInfoView: React.FC = ({ @@ -803,6 +804,7 @@ const TeamInfoView: React.FC = ({