From 09d887dc0cf5212f012986b86338b63b0d88a8b6 Mon Sep 17 00:00:00 2001 From: Mubashir Osmani Date: Thu, 30 Jul 2026 01:25:23 +0000 Subject: [PATCH] feat(ui): offer every logging integration for key/team logging Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/components/callback_info_helpers.tsx | 146 ++++++++++-------- .../components/team/LoggingSettings.test.tsx | 98 +++++++++++- .../src/components/team/LoggingSettings.tsx | 17 +- 3 files changed, 191 insertions(+), 70 deletions(-) diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index 7aa121dcca5..0f3be23e0f1 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -1,8 +1,10 @@ import arizeLogo from "../../public/assets/logos/arize.png"; import awsLogo from "../../public/assets/logos/aws.svg"; +import azureLogo from "../../public/assets/logos/microsoft_azure.svg"; import braintrustLogo from "../../public/assets/logos/braintrust.png"; import datadogLogo from "../../public/assets/logos/datadog.png"; import galileoLogo from "../../public/assets/logos/galileo.ico"; +import googleLogo from "../../public/assets/logos/google.svg"; import lagoLogo from "../../public/assets/logos/lago.svg"; import langfuseLogo from "../../public/assets/logos/langfuse.png"; import langsmithLogo from "../../public/assets/logos/langsmith.png"; @@ -13,7 +15,6 @@ interface CallbackConfig { id: string; displayName: string; logo?: string; - supports_key_team_logging: boolean; dynamic_params: Record; description: string; } @@ -23,76 +24,91 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ id: "arize", displayName: "Arize", logo: arizeLogo.src, - supports_key_team_logging: true, dynamic_params: { arize_api_key: "password", arize_space_id: "password", }, description: "Arize Logging Integration", }, + { + id: "arize_phoenix", + displayName: "Arize Phoenix", + logo: arizeLogo.src, + dynamic_params: {}, + description: "Arize Phoenix Logging Integration", + }, + { + id: "azure_storage", + displayName: "Azure Blob Storage", + logo: azureLogo.src, + dynamic_params: {}, + description: "Azure Blob Storage Logging Integration", + }, { id: "braintrust", displayName: "Braintrust", logo: braintrustLogo.src, - supports_key_team_logging: false, - dynamic_params: { - braintrust_api_key: "password", - braintrust_project_name: "text", - }, + dynamic_params: {}, description: "Braintrust Logging Integration", }, { id: "custom_callback_api", displayName: "Custom Callback API", - supports_key_team_logging: true, - dynamic_params: { - custom_callback_api_url: "text", - custom_callback_api_headers: "text", - }, + dynamic_params: {}, description: "Custom Callback API Logging Integration", }, - { - id: "galileo", - displayName: "Galileo", - logo: galileoLogo.src, - supports_key_team_logging: false, - dynamic_params: { - GALILEO_API_KEY: "password", - GALILEO_PROJECT_ID: "text", - GALILEO_LOG_STREAM_ID: "text", - GALILEO_BASE_URL: "text", - GALILEO_USERNAME: "text", - GALILEO_PASSWORD: "password", - }, - description: "Galileo AI Observability Integration", - }, { id: "datadog", displayName: "Datadog", logo: datadogLogo.src, - supports_key_team_logging: false, dynamic_params: { dd_api_key: "password", dd_site: "text", }, description: "Datadog Logging Integration", }, + { + id: "datadog_llm_observability", + displayName: "Datadog LLM Observability", + logo: datadogLogo.src, + dynamic_params: {}, + description: "Datadog LLM Observability Logging Integration", + }, + { + id: "galileo", + displayName: "Galileo", + logo: galileoLogo.src, + dynamic_params: {}, + description: "Galileo AI Observability Integration", + }, + { + id: "gcs_bucket", + displayName: "GCS Bucket", + logo: googleLogo.src, + dynamic_params: { + gcs_bucket_name: "text", + gcs_path_service_account: "text", + }, + description: "Google Cloud Storage Bucket Logging Integration", + }, + { + id: "gcs_pubsub", + displayName: "GCS Pub/Sub", + logo: googleLogo.src, + dynamic_params: {}, + description: "Google Cloud Pub/Sub Logging Integration", + }, { id: "lago", displayName: "Lago", logo: lagoLogo.src, - supports_key_team_logging: false, - dynamic_params: { - lago_api_url: "text", - lago_api_key: "password", - }, + dynamic_params: {}, description: "Lago Billing Logging Integration", }, { id: "langfuse", displayName: "Langfuse", logo: langfuseLogo.src, - supports_key_team_logging: true, dynamic_params: { langfuse_public_key: "text", langfuse_secret_key: "password", @@ -104,7 +120,6 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ id: "langfuse_otel", displayName: "Langfuse OTEL", logo: langfuseLogo.src, - supports_key_team_logging: true, dynamic_params: { langfuse_public_key: "text", langfuse_secret_key: "password", @@ -116,7 +131,6 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ id: "langsmith", displayName: "LangSmith", logo: langsmithLogo.src, - supports_key_team_logging: true, dynamic_params: { langsmith_api_key: "password", langsmith_project: "text", @@ -125,54 +139,64 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ }, description: "Langsmith Logging Integration", }, + { + id: "mlflow", + displayName: "MLflow", + dynamic_params: {}, + description: "MLflow Logging Integration", + }, { id: "openmeter", displayName: "OpenMeter", logo: openmeterLogo.src, - supports_key_team_logging: false, - dynamic_params: { - openmeter_api_key: "password", - openmeter_base_url: "text", - }, + dynamic_params: {}, description: "OpenMeter Logging Integration", }, + { + id: "opik", + displayName: "Opik", + dynamic_params: {}, + description: "Comet Opik Logging Integration", + }, { id: "otel", displayName: "Open Telemetry", logo: otelLogo.src, - supports_key_team_logging: false, - dynamic_params: { - otel_endpoint: "text", - otel_headers: "text", - }, + dynamic_params: {}, description: "OpenTelemetry Logging Integration", }, + { + id: "posthog", + displayName: "PostHog", + dynamic_params: { + posthog_api_key: "password", + posthog_api_url: "text", + }, + description: "PostHog Logging Integration", + }, { id: "s3", displayName: "S3", logo: awsLogo.src, - supports_key_team_logging: false, - dynamic_params: { - s3_bucket_name: "text", - aws_access_key_id: "password", - aws_secret_access_key: "password", - aws_region: "text", - }, + dynamic_params: {}, description: "S3 Bucket (AWS) Logging Integration", }, { - id: "SQS", + id: "aws_sqs", displayName: "SQS", logo: awsLogo.src, - supports_key_team_logging: false, - dynamic_params: { - sqs_queue_url: "text", - aws_access_key_id: "password", - aws_secret_access_key: "password", - aws_region: "text", - }, + dynamic_params: {}, description: "SQS Queue (AWS) Logging Integration", }, + { + id: "weave_otel", + displayName: "Weave", + dynamic_params: { + wandb_api_key: "password", + weave_project_id: "text", + }, + description: "Weights & Biases Weave Logging Integration", + }, ]; // Create callbackInfo object mapping display names to config objects diff --git a/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx b/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx index e724e32d1c4..ca1da4cbbd5 100644 --- a/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx @@ -1,14 +1,110 @@ import React from "react"; import { describe, it, expect, vi, beforeEach } from "vitest"; import userEvent from "@testing-library/user-event"; -import { renderWithProviders, screen, fireEvent } from "../../../tests/test-utils"; +import { renderWithProviders, screen, fireEvent, within } from "../../../tests/test-utils"; +import { CALLBACK_CONFIGS } from "../callback_info_helpers"; import LoggingSettings from "./LoggingSettings"; +const STANDARD_CALLBACK_DYNAMIC_PARAMS_ACCEPTED_BY_PROXY = [ + "langfuse_public_key", + "langfuse_secret", + "langfuse_secret_key", + "langfuse_host", + "langfuse_prompt_version", + "gcs_bucket_name", + "gcs_path_service_account", + "langsmith_api_key", + "langsmith_project", + "langsmith_base_url", + "langsmith_sampling_rate", + "langsmith_tenant_id", + "humanloop_api_key", + "arize_api_key", + "arize_space_key", + "arize_space_id", + "posthog_api_key", + "posthog_api_url", + "wandb_api_key", + "weave_project_id", + "dd_api_key", + "dd_site", + "dd_agent_host", + "dd_agent_port", + "turn_off_message_logging", + "litellm_disabled_callbacks", +]; + +const openIntegrationTypeDropdown = () => { + const integrationTypeLabel = screen.getByText("Integration Type"); + const select = within(integrationTypeLabel.parentElement as HTMLElement).getByRole("combobox"); + fireEvent.mouseDown(select); +}; + +const integrationOption = (displayName: string): HTMLElement => { + const options = Array.from(document.querySelectorAll(".ant-select-item-option")); + const option = options.find((element) => element.textContent?.endsWith(displayName)); + if (!option) { + throw new Error(`No integration option for ${displayName} in [${options.map((o) => o.textContent).join(", ")}]`); + } + return option as HTMLElement; +}; + describe("LoggingSettings", () => { beforeEach(() => { vi.clearAllMocks(); }); + it("only declares callback vars the proxy accepts for key/team logging", () => { + const declaredParams = CALLBACK_CONFIGS.flatMap((config) => Object.keys(config.dynamic_params)); + + expect(declaredParams.length).toBeGreaterThan(0); + expect( + declaredParams.filter((param) => !STANDARD_CALLBACK_DYNAMIC_PARAMS_ACCEPTED_BY_PROXY.includes(param)), + ).toEqual([]); + }); + + it("offers integrations whose credentials only come from the proxy config", () => { + const initialValue = [{ callback_name: "", callback_type: "success", callback_vars: {} }]; + + renderWithProviders(); + openIntegrationTypeDropdown(); + + for (const displayName of ["Arize Phoenix", "Azure Blob Storage", "Datadog", "Datadog LLM Observability"]) { + expect(integrationOption(displayName)).toBeInTheDocument(); + } + }); + + it("stores the internal callback name when a newly added integration is selected", () => { + const mockOnChange = vi.fn(); + const initialValue = [{ callback_name: "", callback_type: "success", callback_vars: {} }]; + + renderWithProviders(); + openIntegrationTypeDropdown(); + fireEvent.click(integrationOption("Arize Phoenix")); + + expect(mockOnChange).toHaveBeenCalledWith([ + { callback_name: "arize_phoenix", callback_type: "success", callback_vars: {} }, + ]); + }); + + it("renders per-key credential inputs for an integration that supports them", () => { + const initialValue = [{ callback_name: "gcs_bucket", callback_type: "success", callback_vars: {} }]; + + renderWithProviders(); + + expect(screen.getByPlaceholderText("os.environ/GCS_BUCKET_NAME")).toBeInTheDocument(); + expect(screen.getByPlaceholderText("os.environ/GCS_PATH_SERVICE_ACCOUNT")).toBeInTheDocument(); + }); + + it("explains where credentials come from for an integration without per-key credentials", () => { + const initialValue = [{ callback_name: "arize_phoenix", callback_type: "success", callback_vars: {} }]; + + renderWithProviders(); + + expect(screen.queryByText("Integration Parameters")).toBeNull(); + expect(screen.getByText(/reads its credentials from the proxy environment\/config/)).toBeInTheDocument(); + }); + it("passes a number to updateCallbackVar when user inputs a number in NumericalInput", async () => { const user = userEvent.setup(); const mockOnChange = vi.fn(); diff --git a/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx b/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx index 80bea3ef2d2..9d20c292e6a 100644 --- a/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx +++ b/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx @@ -29,12 +29,6 @@ const LoggingSettings: React.FC = ({ disabledCallbacks = [], onDisabledCallbacksChange, }) => { - // Get callbacks that support team and key logging - const supportedCallbacks = Object.entries(callbackInfo) - .filter(([_, info]) => info.supports_key_team_logging) - .map(([name, _]) => name); - - // Get all available callbacks for disabled selection const allCallbacks = Object.keys(callbackInfo); const handleChange = (newValue: LoggingConfig[]) => { @@ -102,7 +96,14 @@ const LoggingSettings: React.FC = ({ const dynamicParams = callbackInfo[callbackDisplayName]?.dynamic_params || {}; - if (Object.keys(dynamicParams).length === 0) return null; + if (Object.keys(dynamicParams).length === 0) { + return ( +
+ {callbackDisplayName} reads its credentials from the proxy environment/config, so there is nothing to fill in + here. +
+ ); + } return (
@@ -268,7 +269,7 @@ const LoggingSettings: React.FC = ({ className="w-full" optionLabelProp="label" > - {supportedCallbacks.map((callbackName) => { + {allCallbacks.map((callbackName) => { const description = callbackInfo[callbackName]?.description; return (