feat(ui): offer every logging integration for key/team logging

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

ui/litellm-dashboard/src/components/callback_info_helpers.tsx

@@ -1,8 +1,10 @@
 import arizeLogo from "../../public/assets/logos/arize.png";
 import awsLogo from "../../public/assets/logos/aws.svg";
+import azureLogo from "../../public/assets/logos/microsoft_azure.svg";
 import braintrustLogo from "../../public/assets/logos/braintrust.png";
 import datadogLogo from "../../public/assets/logos/datadog.png";
 import galileoLogo from "../../public/assets/logos/galileo.ico";
+import googleLogo from "../../public/assets/logos/google.svg";
 import lagoLogo from "../../public/assets/logos/lago.svg";
 import langfuseLogo from "../../public/assets/logos/langfuse.png";
 import langsmithLogo from "../../public/assets/logos/langsmith.png";
@@ -13,7 +15,6 @@ interface CallbackConfig {
   id: string;
   displayName: string;
   logo?: string;
-  supports_key_team_logging: boolean;
   dynamic_params: Record<string, "text" | "password" | "select" | "upload" | "number">;
   description: string;
 }
@@ -23,76 +24,91 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
     id: "arize",
     displayName: "Arize",
     logo: arizeLogo.src,
-    supports_key_team_logging: true,
     dynamic_params: {
       arize_api_key: "password",
       arize_space_id: "password",
     },
     description: "Arize Logging Integration",
   },
+  {
+    id: "arize_phoenix",
+    displayName: "Arize Phoenix",
+    logo: arizeLogo.src,
+    dynamic_params: {},
+    description: "Arize Phoenix Logging Integration",
+  },
+  {
+    id: "azure_storage",
+    displayName: "Azure Blob Storage",
+    logo: azureLogo.src,
+    dynamic_params: {},
+    description: "Azure Blob Storage Logging Integration",
+  },
   {
     id: "braintrust",
     displayName: "Braintrust",
     logo: braintrustLogo.src,
-    supports_key_team_logging: false,
-    dynamic_params: {
-      braintrust_api_key: "password",
-      braintrust_project_name: "text",
-    },
+    dynamic_params: {},
     description: "Braintrust Logging Integration",
   },
   {
     id: "custom_callback_api",
     displayName: "Custom Callback API",
-    supports_key_team_logging: true,
-    dynamic_params: {
-      custom_callback_api_url: "text",
-      custom_callback_api_headers: "text",
-    },
+    dynamic_params: {},
     description: "Custom Callback API Logging Integration",
   },
-  {
-    id: "galileo",
-    displayName: "Galileo",
-    logo: galileoLogo.src,
-    supports_key_team_logging: false,
-    dynamic_params: {
-      GALILEO_API_KEY: "password",
-      GALILEO_PROJECT_ID: "text",
-      GALILEO_LOG_STREAM_ID: "text",
-      GALILEO_BASE_URL: "text",
-      GALILEO_USERNAME: "text",
-      GALILEO_PASSWORD: "password",
-    },
-    description: "Galileo AI Observability Integration",
-  },
   {
     id: "datadog",
     displayName: "Datadog",
     logo: datadogLogo.src,
-    supports_key_team_logging: false,
     dynamic_params: {
       dd_api_key: "password",
       dd_site: "text",
     },
     description: "Datadog Logging Integration",
   },
+  {
+    id: "datadog_llm_observability",
+    displayName: "Datadog LLM Observability",
+    logo: datadogLogo.src,
+    dynamic_params: {},
+    description: "Datadog LLM Observability Logging Integration",
+  },
+  {
+    id: "galileo",
+    displayName: "Galileo",
+    logo: galileoLogo.src,
+    dynamic_params: {},
+    description: "Galileo AI Observability Integration",
+  },
+  {
+    id: "gcs_bucket",
+    displayName: "GCS Bucket",
+    logo: googleLogo.src,
+    dynamic_params: {
+      gcs_bucket_name: "text",
+      gcs_path_service_account: "text",
+    },
+    description: "Google Cloud Storage Bucket Logging Integration",
+  },
+  {
+    id: "gcs_pubsub",
+    displayName: "GCS Pub/Sub",
+    logo: googleLogo.src,
+    dynamic_params: {},
+    description: "Google Cloud Pub/Sub Logging Integration",
+  },
   {
     id: "lago",
     displayName: "Lago",
     logo: lagoLogo.src,
-    supports_key_team_logging: false,
-    dynamic_params: {
-      lago_api_url: "text",
-      lago_api_key: "password",
-    },
+    dynamic_params: {},
     description: "Lago Billing Logging Integration",
   },
   {
     id: "langfuse",
     displayName: "Langfuse",
     logo: langfuseLogo.src,
-    supports_key_team_logging: true,
     dynamic_params: {
       langfuse_public_key: "text",
       langfuse_secret_key: "password",
@@ -104,7 +120,6 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
     id: "langfuse_otel",
     displayName: "Langfuse OTEL",
     logo: langfuseLogo.src,
-    supports_key_team_logging: true,
     dynamic_params: {
       langfuse_public_key: "text",
       langfuse_secret_key: "password",
@@ -116,7 +131,6 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
     id: "langsmith",
     displayName: "LangSmith",
     logo: langsmithLogo.src,
-    supports_key_team_logging: true,
     dynamic_params: {
       langsmith_api_key: "password",
       langsmith_project: "text",
@@ -125,54 +139,64 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
     },
     description: "Langsmith Logging Integration",
   },
+  {
+    id: "mlflow",
+    displayName: "MLflow",
+    dynamic_params: {},
+    description: "MLflow Logging Integration",
+  },
   {
     id: "openmeter",
     displayName: "OpenMeter",
     logo: openmeterLogo.src,
-    supports_key_team_logging: false,
-    dynamic_params: {
-      openmeter_api_key: "password",
-      openmeter_base_url: "text",
-    },
+    dynamic_params: {},
     description: "OpenMeter Logging Integration",
   },
+  {
+    id: "opik",
+    displayName: "Opik",
+    dynamic_params: {},
+    description: "Comet Opik Logging Integration",
+  },
   {
     id: "otel",
     displayName: "Open Telemetry",
     logo: otelLogo.src,
-    supports_key_team_logging: false,
-    dynamic_params: {
-      otel_endpoint: "text",
-      otel_headers: "text",
-    },
+    dynamic_params: {},
     description: "OpenTelemetry Logging Integration",
   },
+  {
+    id: "posthog",
+    displayName: "PostHog",
+    dynamic_params: {
+      posthog_api_key: "password",
+      posthog_api_url: "text",
+    },
+    description: "PostHog Logging Integration",
+  },
   {
     id: "s3",
     displayName: "S3",
     logo: awsLogo.src,
-    supports_key_team_logging: false,
-    dynamic_params: {
-      s3_bucket_name: "text",
-      aws_access_key_id: "password",
-      aws_secret_access_key: "password",
-      aws_region: "text",
-    },
+    dynamic_params: {},
     description: "S3 Bucket (AWS) Logging Integration",
   },
   {
-    id: "SQS",
+    id: "aws_sqs",
     displayName: "SQS",
     logo: awsLogo.src,
-    supports_key_team_logging: false,
-    dynamic_params: {
-      sqs_queue_url: "text",
-      aws_access_key_id: "password",
-      aws_secret_access_key: "password",
-      aws_region: "text",
-    },
+    dynamic_params: {},
     description: "SQS Queue (AWS) Logging Integration",
   },
+  {
+    id: "weave_otel",
+    displayName: "Weave",
+    dynamic_params: {
+      wandb_api_key: "password",
+      weave_project_id: "text",
+    },
+    description: "Weights & Biases Weave Logging Integration",
+  },
 ];
 
 // Create callbackInfo object mapping display names to config objects

ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx

@@ -1,14 +1,110 @@
 import React from "react";
 import { describe, it, expect, vi, beforeEach } from "vitest";
 import userEvent from "@testing-library/user-event";
-import { renderWithProviders, screen, fireEvent } from "../../../tests/test-utils";
+import { renderWithProviders, screen, fireEvent, within } from "../../../tests/test-utils";
+import { CALLBACK_CONFIGS } from "../callback_info_helpers";
 import LoggingSettings from "./LoggingSettings";
 
+const STANDARD_CALLBACK_DYNAMIC_PARAMS_ACCEPTED_BY_PROXY = [
+  "langfuse_public_key",
+  "langfuse_secret",
+  "langfuse_secret_key",
+  "langfuse_host",
+  "langfuse_prompt_version",
+  "gcs_bucket_name",
+  "gcs_path_service_account",
+  "langsmith_api_key",
+  "langsmith_project",
+  "langsmith_base_url",
+  "langsmith_sampling_rate",
+  "langsmith_tenant_id",
+  "humanloop_api_key",
+  "arize_api_key",
+  "arize_space_key",
+  "arize_space_id",
+  "posthog_api_key",
+  "posthog_api_url",
+  "wandb_api_key",
+  "weave_project_id",
+  "dd_api_key",
+  "dd_site",
+  "dd_agent_host",
+  "dd_agent_port",
+  "turn_off_message_logging",
+  "litellm_disabled_callbacks",
+];
+
+const openIntegrationTypeDropdown = () => {
+  const integrationTypeLabel = screen.getByText("Integration Type");
+  const select = within(integrationTypeLabel.parentElement as HTMLElement).getByRole("combobox");
+  fireEvent.mouseDown(select);
+};
+
+const integrationOption = (displayName: string): HTMLElement => {
+  const options = Array.from(document.querySelectorAll(".ant-select-item-option"));
+  const option = options.find((element) => element.textContent?.endsWith(displayName));
+  if (!option) {
+    throw new Error(`No integration option for ${displayName} in [${options.map((o) => o.textContent).join(", ")}]`);
+  }
+  return option as HTMLElement;
+};
+
 describe("LoggingSettings", () => {
   beforeEach(() => {
     vi.clearAllMocks();
   });
 
+  it("only declares callback vars the proxy accepts for key/team logging", () => {
+    const declaredParams = CALLBACK_CONFIGS.flatMap((config) => Object.keys(config.dynamic_params));
+
+    expect(declaredParams.length).toBeGreaterThan(0);
+    expect(
+      declaredParams.filter((param) => !STANDARD_CALLBACK_DYNAMIC_PARAMS_ACCEPTED_BY_PROXY.includes(param)),
+    ).toEqual([]);
+  });
+
+  it("offers integrations whose credentials only come from the proxy config", () => {
+    const initialValue = [{ callback_name: "", callback_type: "success", callback_vars: {} }];
+
+    renderWithProviders(<LoggingSettings value={initialValue} onChange={vi.fn()} />);
+    openIntegrationTypeDropdown();
+
+    for (const displayName of ["Arize Phoenix", "Azure Blob Storage", "Datadog", "Datadog LLM Observability"]) {
+      expect(integrationOption(displayName)).toBeInTheDocument();
+    }
+  });
+
+  it("stores the internal callback name when a newly added integration is selected", () => {
+    const mockOnChange = vi.fn();
+    const initialValue = [{ callback_name: "", callback_type: "success", callback_vars: {} }];
+
+    renderWithProviders(<LoggingSettings value={initialValue} onChange={mockOnChange} />);
+    openIntegrationTypeDropdown();
+    fireEvent.click(integrationOption("Arize Phoenix"));
+
+    expect(mockOnChange).toHaveBeenCalledWith([
+      { callback_name: "arize_phoenix", callback_type: "success", callback_vars: {} },
+    ]);
+  });
+
+  it("renders per-key credential inputs for an integration that supports them", () => {
+    const initialValue = [{ callback_name: "gcs_bucket", callback_type: "success", callback_vars: {} }];
+
+    renderWithProviders(<LoggingSettings value={initialValue} onChange={vi.fn()} />);
+
+    expect(screen.getByPlaceholderText("os.environ/GCS_BUCKET_NAME")).toBeInTheDocument();
+    expect(screen.getByPlaceholderText("os.environ/GCS_PATH_SERVICE_ACCOUNT")).toBeInTheDocument();
+  });
+
+  it("explains where credentials come from for an integration without per-key credentials", () => {
+    const initialValue = [{ callback_name: "arize_phoenix", callback_type: "success", callback_vars: {} }];
+
+    renderWithProviders(<LoggingSettings value={initialValue} onChange={vi.fn()} />);
+
+    expect(screen.queryByText("Integration Parameters")).toBeNull();
+    expect(screen.getByText(/reads its credentials from the proxy environment\/config/)).toBeInTheDocument();
+  });
+
   it("passes a number to updateCallbackVar when user inputs a number in NumericalInput", async () => {
     const user = userEvent.setup();
     const mockOnChange = vi.fn();

ui/litellm-dashboard/src/components/team/LoggingSettings.tsx

@@ -29,12 +29,6 @@ const LoggingSettings: React.FC<LoggingSettingsProps> = ({
   disabledCallbacks = [],
   onDisabledCallbacksChange,
 }) => {
-  // Get callbacks that support team and key logging
-  const supportedCallbacks = Object.entries(callbackInfo)
-    .filter(([_, info]) => info.supports_key_team_logging)
-    .map(([name, _]) => name);
-
-  // Get all available callbacks for disabled selection
   const allCallbacks = Object.keys(callbackInfo);
 
   const handleChange = (newValue: LoggingConfig[]) => {
@@ -102,7 +96,14 @@ const LoggingSettings: React.FC<LoggingSettingsProps> = ({
 
     const dynamicParams = callbackInfo[callbackDisplayName]?.dynamic_params || {};
 
-    if (Object.keys(dynamicParams).length === 0) return null;
+    if (Object.keys(dynamicParams).length === 0) {
+      return (
+        <div className="mt-6 pt-4 border-t border-gray-100 text-xs text-gray-500">
+          {callbackDisplayName} reads its credentials from the proxy environment/config, so there is nothing to fill in
+          here.
+        </div>
+      );
+    }
 
     return (
       <div className="mt-6 pt-4 border-t border-gray-100">
@@ -268,7 +269,7 @@ const LoggingSettings: React.FC<LoggingSettingsProps> = ({
                       className="w-full"
                       optionLabelProp="label"
                     >
-                      {supportedCallbacks.map((callbackName) => {
+                      {allCallbacks.map((callbackName) => {
                         const description = callbackInfo[callbackName]?.description;
                         return (
                           <Option key={callbackName} value={callbackName} label={callbackName}>