From 25cb87346774c6e1c50df85b8af6c49ddc5df765 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 15 Nov 2025 16:44:17 -0800 Subject: [PATCH] [UI] expose backend endpoint for callbacks settings (#16698) * init json for * add SQS fields * fix params * return callbacks configs * test_get_callback_configs * test_get_callback_configs * vercel build fix --- docs/my-website/package-lock.json | 47 +- docs/my-website/package.json | 9 +- litellm/integrations/callback_configs.json | 404 ++++++++++++++++++ .../callback_management_endpoints.py | 30 +- .../test_callback_management_endpoints.py | 37 ++ 5 files changed, 521 insertions(+), 6 deletions(-) create mode 100644 litellm/integrations/callback_configs.json diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json index 3fce178f9aa..67f0ea79e68 100644 --- a/docs/my-website/package-lock.json +++ b/docs/my-website/package-lock.json @@ -11378,9 +11378,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.253", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.253.tgz", - "integrity": "sha512-O0tpQ/35rrgdiGQ0/OFWhy1itmd9A6TY9uQzlqj3hKSu/aYpe7UIn5d7CU2N9myH6biZiWF3VMZVuup8pw5U9w==", + "version": "1.5.254", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.254.tgz", + "integrity": "sha512-DcUsWpVhv9svsKRxnSCZ86SjD+sp32SGidNB37KpqXJncp1mfUgKbHvBomE89WJDbfVKw1mdv5+ikrvd43r+Bg==", "license": "ISC" }, "node_modules/emoji-regex": { @@ -11601,6 +11601,19 @@ "node": ">=8.0.0" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esrecurse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", @@ -12619,6 +12632,28 @@ "node": ">=6.0" } }, + "node_modules/gray-matter/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/gray-matter/node_modules/js-yaml": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/gzip-size": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", @@ -21028,6 +21063,12 @@ "wbuf": "^1.7.3" } }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, "node_modules/srcset": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/srcset/-/srcset-4.0.0.tgz", diff --git a/docs/my-website/package.json b/docs/my-website/package.json index cd67ef6ecbe..784a5e4b578 100644 --- a/docs/my-website/package.json +++ b/docs/my-website/package.json @@ -48,11 +48,16 @@ "node": ">=16.14", "npm": ">=8.3.0" }, + "resolutions": { + "webpack-dev-server": ">=5.2.1", + "form-data": ">=4.0.4", + "mermaid": ">=11.10.0", + "gray-matter": "4.0.3" + }, "overrides": { "webpack-dev-server": ">=5.2.1", "form-data": ">=4.0.4", "mermaid": ">=11.10.0", - "js-yaml": ">=4.1.1", - "gray-matter": ">=4.0.3" + "gray-matter": "4.0.3" } } diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json new file mode 100644 index 00000000000..d8a96e71769 --- /dev/null +++ b/litellm/integrations/callback_configs.json @@ -0,0 +1,404 @@ +[ + { + "id": "arize", + "displayName": "Arize", + "logo": "arize.png", + "supports_key_team_logging": true, + "dynamic_params": { + "arize_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "Arize API key for authentication", + "required": true + }, + "arize_space_key": { + "type": "password", + "ui_name": "Space Key", + "description": "Arize Space key to identify your workspace", + "required": true + } + }, + "description": "Arize Logging Integration" + }, + { + "id": "braintrust", + "displayName": "Braintrust", + "logo": "braintrust.png", + "supports_key_team_logging": false, + "dynamic_params": { + "braintrust_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "Braintrust API key for authentication", + "required": true + }, + "braintrust_project_name": { + "type": "text", + "ui_name": "Project Name", + "description": "Name of the Braintrust project to log to", + "required": true + } + }, + "description": "Braintrust Logging Integration" + }, + { + "id": "custom_callback_api", + "displayName": "Custom Callback API", + "logo": "custom.svg", + "supports_key_team_logging": true, + "dynamic_params": { + "custom_callback_api_url": { + "type": "text", + "ui_name": "Callback URL", + "description": "Your custom webhook/API endpoint URL to receive logs", + "required": true + }, + "custom_callback_api_headers": { + "type": "text", + "ui_name": "Headers (JSON)", + "description": "Custom HTTP headers as JSON string (e.g., {\"Authorization\": \"Bearer token\"})", + "required": false + } + }, + "description": "Custom Callback API Logging Integration" + }, + { + "id": "datadog", + "displayName": "Datadog", + "logo": "datadog.png", + "supports_key_team_logging": false, + "dynamic_params": { + "dd_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "Datadog API key for authentication", + "required": true + }, + "dd_site": { + "type": "text", + "ui_name": "Site", + "description": "Datadog site URL (e.g., us5.datadoghq.com)", + "required": true + } + }, + "description": "Datadog Logging Integration" + }, + { + "id": "lago", + "displayName": "Lago", + "logo": "lago.svg", + "supports_key_team_logging": false, + "dynamic_params": { + "lago_api_url": { + "type": "text", + "ui_name": "API URL", + "description": "Lago API base URL", + "required": true + }, + "lago_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "Lago API key for authentication", + "required": true + } + }, + "description": "Lago Billing Logging Integration" + }, + { + "id": "langfuse", + "displayName": "Langfuse", + "logo": "langfuse.png", + "supports_key_team_logging": true, + "dynamic_params": { + "langfuse_public_key": { + "type": "text", + "ui_name": "Public Key", + "description": "Langfuse public key", + "required": true + }, + "langfuse_secret_key": { + "type": "password", + "ui_name": "Secret Key", + "description": "Langfuse secret key for authentication", + "required": true + }, + "langfuse_host": { + "type": "text", + "ui_name": "Host URL", + "description": "Langfuse host URL (default: https://cloud.langfuse.com)", + "required": false + } + }, + "description": "Langfuse v2 Logging Integration" + }, + { + "id": "langfuse_otel", + "displayName": "Langfuse OTEL", + "logo": "langfuse.png", + "supports_key_team_logging": true, + "dynamic_params": { + "langfuse_public_key": { + "type": "text", + "ui_name": "Public Key", + "description": "Langfuse public key", + "required": true + }, + "langfuse_secret_key": { + "type": "password", + "ui_name": "Secret Key", + "description": "Langfuse secret key for authentication", + "required": true + }, + "langfuse_host": { + "type": "text", + "ui_name": "Host URL", + "description": "Langfuse host URL (default: https://cloud.langfuse.com)", + "required": false + } + }, + "description": "Langfuse v3 OTEL Logging Integration" + }, + { + "id": "langsmith", + "displayName": "LangSmith", + "logo": "langsmith.png", + "supports_key_team_logging": true, + "dynamic_params": { + "langsmith_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "LangSmith API key for authentication", + "required": true + }, + "langsmith_project": { + "type": "text", + "ui_name": "Project Name", + "description": "LangSmith project name (default: litellm-completion)", + "required": false + }, + "langsmith_base_url": { + "type": "text", + "ui_name": "Base URL", + "description": "LangSmith base URL (default: https://api.smith.langchain.com)", + "required": false + }, + "langsmith_sampling_rate": { + "type": "number", + "ui_name": "Sampling Rate", + "description": "Sampling rate for logging (0.0 to 1.0, default: 1.0)", + "required": false + } + }, + "description": "Langsmith Logging Integration" + }, + { + "id": "openmeter", + "displayName": "OpenMeter", + "logo": "openmeter.png", + "supports_key_team_logging": false, + "dynamic_params": { + "openmeter_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "OpenMeter API key for authentication", + "required": true + }, + "openmeter_base_url": { + "type": "text", + "ui_name": "Base URL", + "description": "OpenMeter base URL (default: https://openmeter.cloud)", + "required": false + } + }, + "description": "OpenMeter Logging Integration" + }, + { + "id": "otel", + "displayName": "Open Telemetry", + "logo": "otel.png", + "supports_key_team_logging": false, + "dynamic_params": { + "otel_endpoint": { + "type": "text", + "ui_name": "Endpoint URL", + "description": "OpenTelemetry collector endpoint URL", + "required": true + }, + "otel_headers": { + "type": "text", + "ui_name": "Headers", + "description": "Headers for OTEL exporter (e.g., x-honeycomb-team=YOUR_API_KEY)", + "required": false + } + }, + "description": "OpenTelemetry Logging Integration" + }, + { + "id": "s3", + "displayName": "S3", + "logo": "aws.svg", + "supports_key_team_logging": false, + "dynamic_params": { + "s3_bucket_name": { + "type": "text", + "ui_name": "Bucket Name", + "description": "AWS S3 bucket name to store logs", + "required": true + }, + "s3_region_name": { + "type": "text", + "ui_name": "AWS Region", + "description": "AWS region name (e.g., us-east-1)", + "required": false + }, + "s3_aws_access_key_id": { + "type": "password", + "ui_name": "AWS Access Key ID", + "description": "AWS access key ID for authentication", + "required": false + }, + "s3_aws_secret_access_key": { + "type": "password", + "ui_name": "AWS Secret Access Key", + "description": "AWS secret access key for authentication", + "required": false + }, + "s3_aws_session_token": { + "type": "password", + "ui_name": "AWS Session Token", + "description": "AWS session token for temporary credentials", + "required": false + }, + "s3_endpoint_url": { + "type": "text", + "ui_name": "S3 Endpoint URL", + "description": "Custom S3 endpoint URL (for MinIO or custom S3-compatible services)", + "required": false + }, + "s3_path": { + "type": "text", + "ui_name": "S3 Path Prefix", + "description": "Path prefix within the bucket for organizing logs", + "required": false + } + }, + "description": "S3 Bucket (AWS) Logging Integration" + }, + { + "id": "sqs", + "displayName": "SQS", + "logo": "aws.svg", + "supports_key_team_logging": false, + "dynamic_params": { + "sqs_queue_url": { + "type": "text", + "ui_name": "Queue URL", + "description": "AWS SQS Queue URL", + "required": true + }, + "sqs_region_name": { + "type": "text", + "ui_name": "AWS Region", + "description": "AWS region name (e.g., us-east-1)", + "required": false + }, + "sqs_aws_access_key_id": { + "type": "password", + "ui_name": "AWS Access Key ID", + "description": "AWS access key ID for authentication", + "required": false + }, + "sqs_aws_secret_access_key": { + "type": "password", + "ui_name": "AWS Secret Access Key", + "description": "AWS secret access key for authentication", + "required": false + }, + "sqs_aws_session_token": { + "type": "password", + "ui_name": "AWS Session Token", + "description": "AWS session token for temporary credentials", + "required": false + }, + "sqs_aws_session_name": { + "type": "text", + "ui_name": "AWS Session Name", + "description": "Name for AWS session", + "required": false + }, + "sqs_aws_profile_name": { + "type": "text", + "ui_name": "AWS Profile Name", + "description": "AWS profile name from credentials file", + "required": false + }, + "sqs_aws_role_name": { + "type": "text", + "ui_name": "AWS Role Name", + "description": "AWS IAM role name to assume", + "required": false + }, + "sqs_aws_web_identity_token": { + "type": "password", + "ui_name": "AWS Web Identity Token", + "description": "AWS web identity token for authentication", + "required": false + }, + "sqs_aws_sts_endpoint": { + "type": "text", + "ui_name": "AWS STS Endpoint", + "description": "AWS STS endpoint URL", + "required": false + }, + "sqs_endpoint_url": { + "type": "text", + "ui_name": "SQS Endpoint URL", + "description": "Custom SQS endpoint URL (for LocalStack or custom endpoints)", + "required": false + }, + "sqs_api_version": { + "type": "text", + "ui_name": "API Version", + "description": "SQS API version", + "required": false + }, + "sqs_use_ssl": { + "type": "boolean", + "ui_name": "Use SSL", + "description": "Whether to use SSL for SQS connections", + "required": false + }, + "sqs_verify": { + "type": "boolean", + "ui_name": "Verify SSL", + "description": "Whether to verify SSL certificates", + "required": false + }, + "sqs_strip_base64_files": { + "type": "boolean", + "ui_name": "Strip Base64 Files", + "description": "Remove base64-encoded files from logs to reduce payload size", + "required": false + }, + "sqs_aws_use_application_level_encryption": { + "type": "boolean", + "ui_name": "Use Application-Level Encryption", + "description": "Enable application-level encryption for SQS messages", + "required": false + }, + "sqs_app_encryption_key_b64": { + "type": "password", + "ui_name": "Encryption Key (Base64)", + "description": "Base64-encoded encryption key for application-level encryption", + "required": false + }, + "sqs_app_encryption_aad": { + "type": "text", + "ui_name": "Encryption AAD", + "description": "Additional authenticated data for encryption", + "required": false + } + }, + "description": "SQS Queue (AWS) Logging Integration" + } +] diff --git a/litellm/proxy/management_endpoints/callback_management_endpoints.py b/litellm/proxy/management_endpoints/callback_management_endpoints.py index 3f611715e41..109a5c87d12 100644 --- a/litellm/proxy/management_endpoints/callback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/callback_management_endpoints.py @@ -1,6 +1,10 @@ """ Endpoints for managing callbacks """ +import json +import os +from typing import List + from fastapi import APIRouter, Depends from litellm.litellm_core_utils.logging_callback_manager import CallbacksByType @@ -24,4 +28,28 @@ async def list_callbacks(): # Get callbacks organized by type using the callback manager utility callbacks_by_type = logging_callback_manager.get_callbacks_by_type() - return callbacks_by_type \ No newline at end of file + return callbacks_by_type + + +@router.get( + "/callbacks/configs", + tags=["Logging Callbacks"], + dependencies=[Depends(user_api_key_auth)], +) +async def get_callback_configs(): + """ + Get Available Callback Configurations + + Returns the configuration details for all available logging callbacks, + including supported parameters, field types, and descriptions. + """ + config_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "integrations", + "callback_configs.json" + ) + + with open(config_path, "r") as f: + configs = json.load(f) + + return configs \ No newline at end of file diff --git a/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py index 771534b2aad..c30d6cf0892 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py @@ -215,3 +215,40 @@ class TestCallbackManagementEndpoints: assert key in response_data assert isinstance(response_data[key], list) + def test_get_callback_configs(self): + """Test /callbacks/configs endpoint returns callback configuration JSON""" + # Setup test client + client = TestClient(app) + + # Make request to get callback configs endpoint + response = client.get( + "/callbacks/configs", + headers={"Authorization": "Bearer sk-1234"} + ) + + # Verify response + assert response.status_code == 200 + + response_data = response.json() + + # Verify response is a list + assert isinstance(response_data, list) + + # Verify it contains callback configurations + assert len(response_data) > 0 + + # Verify structure of first callback config + first_config = response_data[0] + assert "id" in first_config + assert "displayName" in first_config + assert "logo" in first_config + assert "supports_key_team_logging" in first_config + assert "dynamic_params" in first_config + assert "description" in first_config + + # Verify dynamic_params structure + assert isinstance(first_config["dynamic_params"], dict) + + + assert has_detailed_params, "Expected at least one callback to have detailed parameter configuration" +