[Feat] UI - Allow adding team specific logging callbacks (#12261)

* add arize logo

* use correct struct

* define dynamic params

* add arize_space_id

* update ui

* ui - team logging

litellm/types/utils.py

@@ -2077,6 +2077,7 @@ class StandardCallbackDynamicParams(TypedDict, total=False):
     # Arize dynamic params
     arize_api_key: Optional[str]
     arize_space_key: Optional[str]
+    arize_space_id: Optional[str]
 
     # Logging settings
     turn_off_message_logging: Optional[bool]  # when true will not log messages

ui/litellm-dashboard/src/components/callback_info_helpers.tsx

@@ -8,6 +8,7 @@ export enum Callbacks {
   OpenMeter = "OpenMeter",
   OTel = "Open Telemetry",
   S3 = "S3",
+  Arize = "Arize",
 }
 
 export const callback_map: Record<string, string> = {
@@ -20,18 +21,74 @@ export const callback_map: Record<string, string> = {
   OpenMeter: "openmeter",
   OTel: "otel",
   S3: "s3",
+  Arize: "arize",
 }
 
 const asset_logos_folder = '/ui/assets/logos/';
 
-export const callbackLogoMap: Record<string, string> = {
-    [Callbacks.Braintrust]: `${asset_logos_folder}braintrust.png`,
-    [Callbacks.Datadog]: `${asset_logos_folder}datadog.png`,
-    [Callbacks.Langfuse]: `${asset_logos_folder}langfuse.png`,
-    [Callbacks.LangSmith]: `${asset_logos_folder}langsmith.png`,
-    [Callbacks.Lago]: `${asset_logos_folder}lago.svg`,
-    [Callbacks.OpenMeter]: `${asset_logos_folder}openmeter.png`,
-    [Callbacks.OTel]: `${asset_logos_folder}otel.png`,
-    [Callbacks.S3]: `${asset_logos_folder}aws.svg`,
-    [Callbacks.CustomCallbackAPI]: `${asset_logos_folder}custom.svg`,
+interface CallbackInfo {
+  logo: string;
+  supports_key_team_logging: boolean;
+  dynamic_params: Record<string, "text" | "password" | "select" | "upload">;
+}
+
+export const callbackInfo: Record<string, CallbackInfo> = {
+  [Callbacks.Langfuse]: {
+    logo: `${asset_logos_folder}langfuse.png`,
+    supports_key_team_logging: true,
+    dynamic_params: {
+        "langfuse_public_key": "text",
+        "langfuse_secret_key": "password",
+        "langfuse_host": "text"
+    }
+    },
+    [Callbacks.Arize]: {
+      logo: `${asset_logos_folder}arize.png`,
+      supports_key_team_logging: true,
+      dynamic_params: {
+        "arize_api_key": "password",
+        "arize_space_id": "text",
+      }
+    },
+    [Callbacks.Braintrust]: {
+        logo: `${asset_logos_folder}braintrust.png`,
+        supports_key_team_logging: false,
+        dynamic_params: {}
+    },
+    [Callbacks.CustomCallbackAPI]: {
+        logo: `${asset_logos_folder}custom.svg`,
+        supports_key_team_logging: false,
+        dynamic_params: {}
+    },
+    [Callbacks.Datadog]: {
+        logo: `${asset_logos_folder}datadog.png`,
+        supports_key_team_logging: false,
+        dynamic_params: {}
+    },
+    [Callbacks.LangSmith]: {
+        logo: `${asset_logos_folder}langsmith.png`,
+        supports_key_team_logging: false,
+        dynamic_params: {}
+    },
+    [Callbacks.Lago]: {
+        logo: `${asset_logos_folder}lago.svg`,
+        supports_key_team_logging: false,
+        dynamic_params: {}
+    },
+    [Callbacks.OpenMeter]: {
+        logo: `${asset_logos_folder}openmeter.png`,
+        supports_key_team_logging: false,
+        dynamic_params: {}
+    },
+    [Callbacks.OTel]: {
+        logo: `${asset_logos_folder}otel.png`,
+        supports_key_team_logging: false,
+        dynamic_params: {}
+    },
+    [Callbacks.S3]: {
+        logo: `${asset_logos_folder}aws.svg`,
+        supports_key_team_logging: false,
+        dynamic_params: {}
+    }
 };
+  

ui/litellm-dashboard/src/components/settings.tsx

@@ -46,7 +46,7 @@ import AlertingSettings from "./alerting/alerting_settings";
 import FormItem from "antd/es/form/FormItem";
 import {
   callback_map,
-  callbackLogoMap,
+  callbackInfo,
   Callbacks,
 } from "./callback_info_helpers";
 interface SettingsPageProps {
@@ -620,28 +620,16 @@ const Settings: React.FC<SettingsPageProps> = ({
                       value={callback_map[callbackEnum]}
                     >
                       <div className="flex items-center space-x-2">
-                        {callbackLogoMap[callbackDisplayName] ? (
+                        {callbackInfo[callbackDisplayName]?.logo ? (
                           <div className="w-5 h-5 flex items-center justify-center">
                             <img
-                              src={callbackLogoMap[callbackDisplayName]}
+                              src={callbackInfo[callbackDisplayName].logo}
                               alt={`${callbackEnum} logo`}
                               className="w-5 h-5"
-                              onError={() => {
-                                if (callbackLogoMap[callbackDisplayName]) {
-                                  delete callbackLogoMap[callbackDisplayName];
-                                }
-                              }}
-                              style={{
-                                display: callbackLogoMap[callbackDisplayName]
-                                  ? "block"
-                                  : "none",
+                              onError={(e) => {
+                                e.currentTarget.style.display = 'none';
                               }}
                             />
-                            {!callbackLogoMap[callbackDisplayName] && (
-                              <div className="w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs">
-                                {callbackDisplayName.charAt(0).toUpperCase()}
-                              </div>
-                            )}
                           </div>
                         ) : (
                           <div className="w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs">

ui/litellm-dashboard/src/components/team/LoggingSettings.tsx

@@ -0,0 +1,259 @@
+import React, { useState } from 'react';
+import { Form, Select, Space, Tooltip, Divider } from 'antd';
+import { InfoCircleOutlined } from '@ant-design/icons';
+import { Button, Card } from '@tremor/react';
+import { PlusIcon, TrashIcon, CogIcon } from '@heroicons/react/outline';
+import { callbackInfo, Callbacks, callback_map } from '../callback_info_helpers';
+
+const { Option } = Select;
+
+interface LoggingConfig {
+  callback_name: string;
+  callback_type: string;
+  callback_vars: Record<string, string>;
+}
+
+interface LoggingSettingsProps {
+  value?: LoggingConfig[];
+  onChange?: (value: LoggingConfig[]) => void;
+}
+
+const LoggingSettings: React.FC<LoggingSettingsProps> = ({ value = [], onChange }) => {
+  // Get callbacks that support team and key logging
+  const supportedCallbacks = Object.entries(callbackInfo)
+    .filter(([_, info]) => info.supports_key_team_logging)
+    .map(([name, _]) => name);
+
+  const handleChange = (newValue: LoggingConfig[]) => {
+    onChange?.(newValue);
+  };
+
+  const addLoggingConfig = () => {
+    const newConfig: LoggingConfig = {
+      callback_name: '',
+      callback_type: 'success',
+      callback_vars: {}
+    };
+    handleChange([...value, newConfig]);
+  };
+
+  const removeLoggingConfig = (index: number) => {
+    const newValue = value.filter((_, i) => i !== index);
+    handleChange(newValue);
+  };
+
+  const updateLoggingConfig = (index: number, field: keyof LoggingConfig, newValue: any) => {
+    const updatedConfigs = [...value];
+    if (field === 'callback_name') {
+      // Convert display name to callback value and reset callback_vars when callback changes
+      const callbackValue = callback_map[newValue] || newValue;
+      updatedConfigs[index] = {
+        ...updatedConfigs[index],
+        [field]: callbackValue,
+        callback_vars: {}
+      };
+    } else {
+      updatedConfigs[index] = {
+        ...updatedConfigs[index],
+        [field]: newValue
+      };
+    }
+    handleChange(updatedConfigs);
+  };
+
+  const updateCallbackVar = (configIndex: number, varName: string, varValue: string) => {
+    const updatedConfigs = [...value];
+    updatedConfigs[configIndex] = {
+      ...updatedConfigs[configIndex],
+      callback_vars: {
+        ...updatedConfigs[configIndex].callback_vars,
+        [varName]: varValue
+      }
+    };
+    handleChange(updatedConfigs);
+  };
+
+  const renderDynamicParams = (config: LoggingConfig, configIndex: number) => {
+    if (!config.callback_name) return null;
+
+    // Find the display name for the callback
+    const callbackDisplayName = Object.entries(callback_map).find(
+      ([_, value]) => value === config.callback_name
+    )?.[0];
+
+    if (!callbackDisplayName) return null;
+
+    const dynamicParams = callbackInfo[callbackDisplayName]?.dynamic_params || {};
+
+    if (Object.keys(dynamicParams).length === 0) return null;
+
+    return (
+      <div className="mt-6 pt-4 border-t border-gray-100">
+        <div className="flex items-center space-x-2 mb-4">
+          <div className="w-3 h-3 bg-blue-100 rounded-full flex items-center justify-center">
+            <div className="w-1.5 h-1.5 bg-blue-500 rounded-full"></div>
+          </div>
+          <span className="text-sm font-medium text-gray-700">Integration Parameters</span>
+        </div>
+        <div className="grid grid-cols-1 gap-4">
+          {Object.entries(dynamicParams).map(([paramName, paramType]) => (
+            <div key={paramName} className="space-y-2">
+              <label className="text-sm font-medium text-gray-700 capitalize flex items-center space-x-1">
+                <span>{paramName.replace(/_/g, ' ')}</span>
+                <Tooltip title={`Environment variable reference recommended: os.environ/${paramName.toUpperCase()}`}>
+                  <InfoCircleOutlined className="text-gray-400 cursor-help text-xs" />
+                </Tooltip>
+                {paramType === 'password' && (
+                  <span className="inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-yellow-100 text-yellow-800">
+                    Sensitive
+                  </span>
+                )}
+              </label>
+              <input
+                type={paramType === 'password' ? 'password' : 'text'}
+                className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500 transition-colors duration-200"
+                placeholder={`os.environ/${paramName.toUpperCase()}`}
+                value={config.callback_vars[paramName] || ''}
+                onChange={(e) => updateCallbackVar(configIndex, paramName, e.target.value)}
+              />
+            </div>
+          ))}
+        </div>
+      </div>
+    );
+  };
+
+  return (
+    <div className="space-y-6">
+      <div className="flex justify-between items-center">
+        <div className="flex items-center space-x-2">
+          <CogIcon className="w-5 h-5 text-blue-500" />
+          <span className="text-base font-semibold text-gray-800">Logging Integrations</span>
+          <Tooltip title="Configure callback logging integrations for this team.">
+            <InfoCircleOutlined className="text-gray-400 cursor-help" />
+          </Tooltip>
+        </div>
+        <Button
+          variant="secondary"
+          onClick={addLoggingConfig}
+          icon={PlusIcon}
+          size="sm"
+          className="hover:border-blue-400 hover:text-blue-500"
+        >
+          Add Integration
+        </Button>
+      </div>
+
+      <div className="space-y-4">
+        {value.map((config, index) => {
+          const callbackDisplayName = config.callback_name 
+            ? Object.entries(callback_map).find(([_, value]) => value === config.callback_name)?.[0] 
+            : undefined;
+          const logoUrl = callbackDisplayName ? callbackInfo[callbackDisplayName]?.logo : null;
+
+          return (
+            <Card
+              key={index}
+              className="border border-gray-200 shadow-sm hover:shadow-md transition-shadow duration-200"
+              decoration="top"
+              decorationColor="blue"
+            >
+              <div className="flex justify-between items-start mb-4">
+                <div className="flex items-center space-x-2">
+                  {logoUrl && (
+                    <img src={logoUrl} alt={callbackDisplayName} className="w-5 h-5 object-contain" />
+                  )}
+                  <span className="text-sm font-medium">
+                    {callbackDisplayName || 'New Integration'} Configuration
+                  </span>
+                </div>
+                <Button
+                  variant="light"
+                  onClick={() => removeLoggingConfig(index)}
+                  icon={TrashIcon}
+                  size="xs"
+                  color="red"
+                  className="hover:bg-red-50"
+                >
+                  Remove
+                </Button>
+              </div>
+              <div className="space-y-4">
+                <div className="grid grid-cols-1 md:grid-cols-2 gap-4">
+                  <div className="space-y-2">
+                    <label className="text-sm font-medium text-gray-700">Integration Type</label>
+                    <Select
+                      value={callbackDisplayName}
+                      placeholder="Select integration"
+                      onChange={(value) => updateLoggingConfig(index, 'callback_name', value)}
+                      className="w-full"
+                      optionLabelProp="label"
+                    >
+                      {supportedCallbacks.map((callbackName) => {
+                        const logo = callbackInfo[callbackName]?.logo;
+                        return (
+                          <Option key={callbackName} value={callbackName} label={callbackName}>
+                            <div className="flex items-center space-x-2">
+                              {logo && (
+                                <img 
+                                  src={logo} 
+                                  alt={callbackName} 
+                                  className="w-4 h-4 object-contain" 
+                                />
+                              )}
+                              <span>{callbackName}</span>
+                            </div>
+                          </Option>
+                        );
+                      })}
+                    </Select>
+                  </div>
+
+                  <div className="space-y-2">
+                    <label className="text-sm font-medium text-gray-700">Event Type</label>
+                    <Select
+                      value={config.callback_type}
+                      onChange={(value) => updateLoggingConfig(index, 'callback_type', value)}
+                      className="w-full"
+                    >
+                      <Option value="success">
+                        <div className="flex items-center space-x-2">
+                          <div className="w-2 h-2 bg-green-500 rounded-full"></div>
+                          <span>Success Only</span>
+                        </div>
+                      </Option>
+                      <Option value="failure">
+                        <div className="flex items-center space-x-2">
+                          <div className="w-2 h-2 bg-red-500 rounded-full"></div>
+                          <span>Failure Only</span>
+                        </div>
+                      </Option>
+                      <Option value="success_and_failure">
+                        <div className="flex items-center space-x-2">
+                          <div className="w-2 h-2 bg-blue-500 rounded-full"></div>
+                          <span>Success & Failure</span>
+                        </div>
+                      </Option>
+                    </Select>
+                  </div>
+                </div>
+
+                {renderDynamicParams(config, index)}
+              </div>
+            </Card>
+          );
+        })}
+      </div>
+
+      {value.length === 0 && (
+        <div className="text-center py-12 text-gray-500 border-2 border-dashed border-gray-200 rounded-lg bg-gray-50/50">
+          <CogIcon className="w-12 h-12 text-gray-300 mb-3 mx-auto" />
+          <div className="text-base font-medium mb-1">No logging integrations configured</div>
+          <div className="text-sm text-gray-400">Click "Add Integration" to configure logging for this team</div>
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default LoggingSettings; 

ui/litellm-dashboard/src/components/teams.tsx

@@ -60,6 +60,7 @@ import AvailableTeamsPanel from "@/components/team/available_teams";
 import VectorStoreSelector from "./vector_store_management/VectorStoreSelector";
 import PremiumVectorStoreSelector from "./common_components/PremiumVectorStoreSelector";
 import PremiumMCPSelector from "./common_components/PremiumMCPSelector";
+import LoggingSettings from "./team/LoggingSettings";
 import type { KeyResponse, Team } from "./key_team_helpers/key_list";
 
 interface TeamProps {
@@ -181,6 +182,7 @@ const Teams: React.FC<TeamProps> = ({
   // Add this state near the other useState declarations
   const [guardrailsList, setGuardrailsList] = useState<string[]>([]);
   const [expandedAccordions, setExpandedAccordions] = useState<Record<string, boolean>>({});
+  const [loggingSettings, setLoggingSettings] = useState<any[]>([]);
 
   useEffect(() => {
     console.log(`currentOrgForCreateTeam: ${currentOrgForCreateTeam}`);
@@ -234,6 +236,7 @@ const Teams: React.FC<TeamProps> = ({
   const handleOk = () => {
     setIsTeamModalVisible(false);
     form.resetFields();
+    setLoggingSettings([]);
   };
 
   const handleMemberOk = () => {
@@ -244,8 +247,8 @@ const Teams: React.FC<TeamProps> = ({
 
   const handleCancel = () => {
     setIsTeamModalVisible(false);
-
     form.resetFields();
+    setLoggingSettings([]);
   };
 
   const handleMemberCancel = () => {
@@ -325,6 +328,27 @@ const Teams: React.FC<TeamProps> = ({
         }
 
         message.info("Creating Team");
+        
+        // Handle logging settings in metadata
+        if (loggingSettings.length > 0) {
+          let metadata = {};
+          if (formValues.metadata) {
+            try {
+              metadata = JSON.parse(formValues.metadata);
+            } catch (e) {
+              console.warn("Invalid JSON in metadata field, starting with empty object");
+            }
+          }
+          
+          // Add logging settings to metadata
+          metadata = {
+            ...metadata,
+            logging: loggingSettings.filter(config => config.callback_name) // Only include configs with callback_name
+          };
+          
+          formValues.metadata = JSON.stringify(metadata);
+        }
+        
         // Transform allowed_vector_store_ids and allowed_mcp_server_ids into object_permission
         if (formValues.allowed_vector_store_ids && formValues.allowed_vector_store_ids.length > 0) {
           formValues.object_permission = {
@@ -350,6 +374,7 @@ const Teams: React.FC<TeamProps> = ({
         console.log(`response for team create call: ${response}`);
         message.success("Team created");
         form.resetFields();
+        setLoggingSettings([]);
         setIsTeamModalVisible(false);
       }
     } catch (error) {
@@ -911,7 +936,7 @@ const Teams: React.FC<TeamProps> = ({
                 <Modal
                   title="Create Team"
                   visible={isTeamModalVisible}
-                  width={800}
+                  width={1000}
                   footer={null}
                   onOk={handleOk}
                   onCancel={handleCancel}
@@ -1145,6 +1170,20 @@ const Teams: React.FC<TeamProps> = ({
                           </Form.Item>
                         </AccordionBody>
                       </Accordion>
+
+                      <Accordion className="mt-8 mb-8">
+                        <AccordionHeader>
+                          <b>Logging Settings</b>
+                        </AccordionHeader>
+                        <AccordionBody>
+                          <div className="mt-4">
+                            <LoggingSettings
+                              value={loggingSettings}
+                              onChange={setLoggingSettings}
+                            />
+                          </div>
+                        </AccordionBody>
+                      </Accordion>
                     </>
                     <div style={{ textAlign: "right", marginTop: "10px" }}>
                       <Button2 htmlType="submit">Create Team</Button2>