fix: add arize from ui

This commit is contained in:
mubashir1osmani 2025-10-15 22:07:16 -04:00
parent 3f8c4598da
commit 0c4aae0347
5 changed files with 482 additions and 200 deletions

View file

@ -103,7 +103,39 @@ class ArizeLogger(OpenTelemetry):
):
"""Arize is used mainly for LLM I/O tracing, sending Proxy Server Request adds bloat to arize logs"""
pass
async def async_health_check(self):
"""
Performs a health check for Arize integration.
Returns:
dict: Health check result with status and message
"""
try:
config = self.get_arize_config()
if not config.space_key:
return {
"status": "unhealthy",
"error_message": "ARIZE_SPACE_KEY environment variable not set"
}
if not config.api_key:
return {
"status": "unhealthy",
"error_message": "ARIZE_API_KEY environment variable not set"
}
return {
"status": "healthy",
"message": "Arize credentials are configured properly"
}
except Exception as e:
return {
"status": "unhealthy",
"error_message": f"Arize health check failed: {str(e)}"
}
def construct_dynamic_otel_headers(
self,

View file

@ -34,7 +34,7 @@ from litellm.proxy.health_check import (
#### Health ENDPOINTS ####
router = APIRouter()
services = Union[Literal["slack_budget_alerts", "langfuse", "slack", "openmeter", "webhook", "email", "braintrust", "datadog", "generic_api", "arize"], str]
@router.get(
"/test",
@ -64,20 +64,7 @@ async def test_endpoint(request: Request):
)
async def health_services_endpoint( # noqa: PLR0915
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
service: Union[
Literal[
"slack_budget_alerts",
"langfuse",
"slack",
"openmeter",
"webhook",
"email",
"braintrust",
"datadog",
"generic_api",
],
str,
] = fastapi.Query(description="Specify the service being hit."),
service: services = fastapi.Query(description="Specify the service being hit."),
):
"""
Use this admin-only endpoint to check if the service is healthy.
@ -113,11 +100,12 @@ async def health_services_endpoint( # noqa: PLR0915
"langsmith",
"datadog",
"generic_api",
"arize",
]:
raise HTTPException(
status_code=400,
detail={
"error": f"Service must be in list. Service={service}. List={['slack_budget_alerts']}"
"error": f"Service must be in list. Service={service} not in {services}"
},
)
@ -150,6 +138,19 @@ async def health_services_endpoint( # noqa: PLR0915
else "Datadog is healthy"
),
}
elif service == "arize":
from litellm.integrations.arize.arize import ArizeLogger
arize_logger = ArizeLogger()
response = await arize_logger.async_health_check()
return {
"status": response["status"],
"message": (
response["error_message"]
if response["status"] == "unhealthy"
else "Arize is healthy"
),
}
elif service == "langfuse":
from litellm.integrations.langfuse.langfuse import LangFuseLogger

View file

@ -0,0 +1,183 @@
"""
Test Arize health check functionality and proxy integration.
"""
import json
import os
import sys
from unittest.mock import patch, MagicMock
# Adds the grandparent directory to sys.path to allow importing project modules
sys.path.insert(0, os.path.abspath("../.."))
import asyncio
import pytest
import litellm
from litellm.integrations.arize.arize import ArizeLogger
from litellm.types.utils import StandardCallbackDynamicParams
class TestArizeHealthCheck:
"""Test Arize health check functionality."""
@pytest.mark.asyncio
async def test_arize_health_check_with_credentials(self):
"""Test Arize health check returns healthy when credentials are available."""
with patch.dict(os.environ, {
"ARIZE_SPACE_KEY": "test-space-key",
"ARIZE_API_KEY": "test-api-key",
"ARIZE_ENDPOINT": "https://otlp.arize.com/v1"
}):
arize_logger = ArizeLogger()
response = await arize_logger.async_health_check()
assert response["status"] == "healthy"
assert "configured properly" in response["message"]
@pytest.mark.asyncio
async def test_arize_health_check_missing_space_key(self):
"""Test Arize health check returns unhealthy when space key is missing."""
with patch.dict(os.environ, {
"ARIZE_API_KEY": "test-api-key"
}, clear=True):
arize_logger = ArizeLogger()
response = await arize_logger.async_health_check()
assert response["status"] == "unhealthy"
assert "ARIZE_SPACE_KEY" in response["error_message"]
@pytest.mark.asyncio
async def test_arize_health_check_missing_api_key(self):
"""Test Arize health check returns unhealthy when API key is missing."""
with patch.dict(os.environ, {
"ARIZE_SPACE_KEY": "test-space-key"
}, clear=True):
arize_logger = ArizeLogger()
response = await arize_logger.async_health_check()
assert response["status"] == "unhealthy"
assert "ARIZE_API_KEY" in response["error_message"]
@pytest.mark.asyncio
async def test_arize_health_check_missing_both_keys(self):
"""Test Arize health check when both keys are missing."""
with patch.dict(os.environ, {}, clear=True):
arize_logger = ArizeLogger()
response = await arize_logger.async_health_check()
assert response["status"] == "unhealthy"
assert "ARIZE_SPACE_KEY" in response["error_message"]
class TestArizeIntegrationWithProxy:
"""Test Arize integration with LiteLLM completion requests."""
@pytest.mark.asyncio
async def test_arize_logging_with_completion(self):
"""Test that Arize logging works with actual completion requests."""
with patch.dict(os.environ, {
"ARIZE_SPACE_KEY": "test-space-key",
"ARIZE_API_KEY": "test-api-key",
"ARIZE_ENDPOINT": "https://otlp.arize.com/v1"
}):
# Create ArizeLogger instance
arize_logger = ArizeLogger()
# Store original callbacks
original_callbacks = litellm.success_callback.copy() if litellm.success_callback else []
try:
# Add ArizeLogger to callbacks
litellm.success_callback = [arize_logger]
# Make completion request
response = await litellm.acompletion(
model="openai/litellm-mock-response-model",
messages=[{"role": "user", "content": "Test message for Arize health check"}],
mock_response="This is a test response that validates Arize integration.",
user="test-arize-health"
)
# Verify response is valid
assert response is not None
print(f"Response type: {type(response)}")
print("✅ Arize completion request completed successfully")
# Give time for async logging
await asyncio.sleep(0.1)
print("✅ Arize completion logging test successful")
finally:
# Restore original callbacks
litellm.success_callback = original_callbacks
def test_arize_get_config(self):
"""Test ArizeLogger.get_arize_config() method."""
with patch.dict(os.environ, {
"ARIZE_SPACE_KEY": "test-space-123",
"ARIZE_API_KEY": "test-api-456",
"ARIZE_ENDPOINT": "https://custom.arize.com/v1"
}):
config = ArizeLogger.get_arize_config()
assert config.space_key == "test-space-123"
assert config.api_key == "test-api-456"
assert config.endpoint == "https://custom.arize.com/v1"
assert config.protocol == "otlp_grpc"
def test_arize_get_config_defaults(self):
"""Test ArizeLogger.get_arize_config() with default endpoint."""
with patch.dict(os.environ, {
"ARIZE_SPACE_KEY": "test-space-default",
"ARIZE_API_KEY": "test-api-default"
}, clear=True):
config = ArizeLogger.get_arize_config()
assert config.space_key == "test-space-default"
assert config.api_key == "test-api-default"
assert config.endpoint == "https://otlp.arize.com/v1" # Default endpoint
assert config.protocol == "otlp_grpc" # Default protocol
def test_arize_construct_dynamic_headers(self):
"""Test dynamic OTEL headers construction for team/key logging."""
arize_logger = ArizeLogger()
dynamic_params = StandardCallbackDynamicParams(
arize_space_key="dynamic-space-123",
arize_api_key="dynamic-api-456"
)
headers = arize_logger.construct_dynamic_otel_headers(dynamic_params)
assert headers is not None
assert headers["arize-space-id"] == "dynamic-space-123"
assert headers["api_key"] == "dynamic-api-456"
def test_arize_construct_dynamic_headers_space_id_fallback(self):
"""Test dynamic headers with arize_space_id parameter (fallback)."""
arize_logger = ArizeLogger()
dynamic_params = StandardCallbackDynamicParams(
arize_space_id="fallback-space-789", # Using space_id instead of space_key
arize_api_key="fallback-api-999"
)
headers = arize_logger.construct_dynamic_otel_headers(dynamic_params)
assert headers is not None
assert headers["arize-space-id"] == "fallback-space-789"
assert headers["api_key"] == "fallback-api-999"
if __name__ == "__main__":
pytest.main([__file__, "-v"])

View file

@ -1,137 +1,151 @@
export enum Callbacks {
Braintrust = "Braintrust",
CustomCallbackAPI = "Custom Callback API",
Datadog = "Datadog",
Langfuse = "Langfuse",
LangfuseOtel = "LangfuseOtel",
LangSmith = "LangSmith",
Lago = "Lago",
OpenMeter = "OpenMeter",
OTel = "Open Telemetry",
S3 = "S3",
Arize = "Arize",
}
export const callback_map: Record<string, string> = {
Braintrust: "braintrust",
CustomCallbackAPI: "custom_callback_api",
Datadog: "datadog",
Langfuse: "langfuse",
LangfuseOtel: "langfuse_otel",
LangSmith: "langsmith",
Lago: "lago",
OpenMeter: "openmeter",
OTel: "otel",
S3: "s3",
Arize: "arize",
}
// Reverse mapping from internal values to display names
export const reverse_callback_map: Record<string, string> = Object.fromEntries(
Object.entries(callback_map).map(([key, value]) => [value, key])
);
// Utility function to convert internal callback values to display names
export const mapInternalToDisplayNames = (internalValues: string[]): string[] => {
return internalValues.map(value => reverse_callback_map[value] || value);
};
// Utility function to convert display names to internal callback values
export const mapDisplayToInternalNames = (displayValues: string[]): string[] => {
return displayValues.map(value => callback_map[value] || value);
};
const asset_logos_folder = '/ui/assets/logos/';
interface CallbackInfo {
logo: string;
interface CallbackConfig {
id: string; // Internal callback name (e.g., "arize", "custom_callback_api")
displayName: string; // User-facing name (e.g., "Arize", "Custom Callback API")
logo: string; // Logo path
supports_key_team_logging: boolean;
dynamic_params: Record<string, "text" | "password" | "select" | "upload" | "number">;
description: string | null;
description: string;
}
export const callbackInfo: Record<string, CallbackInfo> = {
[Callbacks.Langfuse]: {
// Single source of truth for ALL callback configurations
export const CALLBACK_CONFIGS: CallbackConfig[] = [
{
id: "arize",
displayName: "Arize",
logo: `${asset_logos_folder}arize.png`,
supports_key_team_logging: true,
dynamic_params: {
"arize_api_key": "password",
"arize_space_key": "password",
},
description: "Arize Logging Integration"
},
{
id: "braintrust",
displayName: "Braintrust",
logo: `${asset_logos_folder}braintrust.png`,
supports_key_team_logging: false,
dynamic_params: {
"braintrust_api_key": "password",
"braintrust_project_name": "text"
},
description: "Braintrust Logging Integration"
},
{
id: "custom_callback_api",
displayName: "Custom Callback API",
logo: `${asset_logos_folder}custom.svg`,
supports_key_team_logging: true,
dynamic_params: {
"custom_callback_api_url": "text",
"custom_callback_api_headers": "text"
},
description: "Custom Callback API Logging Integration"
},
{
id: "datadog",
displayName: "Datadog",
logo: `${asset_logos_folder}datadog.png`,
supports_key_team_logging: false,
dynamic_params: {
"dd_api_key": "password",
"dd_site": "text"
},
description: "Datadog Logging Integration"
},
{
id: "lago",
displayName: "Lago",
logo: `${asset_logos_folder}lago.svg`,
supports_key_team_logging: false,
dynamic_params: {
"lago_api_url": "text",
"lago_api_key": "password"
},
description: "Lago Billing Logging Integration"
},
{
id: "langfuse",
displayName: "Langfuse",
logo: `${asset_logos_folder}langfuse.png`,
supports_key_team_logging: true,
dynamic_params: {
"langfuse_public_key": "text",
"langfuse_secret_key": "password",
"langfuse_host": "text"
"langfuse_public_key": "text",
"langfuse_secret_key": "password",
"langfuse_host": "text"
},
description: "Langfuse v2 Logging Integration"
},
{
id: "langfuse_otel",
displayName: "Langfuse OTEL",
logo: `${asset_logos_folder}langfuse.png`,
supports_key_team_logging: true,
dynamic_params: {
"langfuse_public_key": "text",
"langfuse_secret_key": "password",
"langfuse_host": "text"
},
[Callbacks.LangfuseOtel]: {
logo: `${asset_logos_folder}langfuse.png`,
supports_key_team_logging: true,
dynamic_params: {
"langfuse_public_key": "text",
"langfuse_secret_key": "password",
"langfuse_host": "text"
},
description: "Langfuse v3 OTEL Logging Integration"
description: "Langfuse v3 OTEL Logging Integration"
},
{
id: "langsmith",
displayName: "LangSmith",
logo: `${asset_logos_folder}langsmith.png`,
supports_key_team_logging: true,
dynamic_params: {
"langsmith_api_key": "password",
"langsmith_project": "text",
"langsmith_base_url": "text",
"langsmith_sampling_rate": "number"
},
[Callbacks.Arize]: {
logo: `${asset_logos_folder}arize.png`,
supports_key_team_logging: true,
dynamic_params: {
"arize_api_key": "password",
"arize_space_id": "text",
},
description: "Arize Logging Integration"
description: "Langsmith Logging Integration"
},
{
id: "openmeter",
displayName: "OpenMeter",
logo: `${asset_logos_folder}openmeter.png`,
supports_key_team_logging: false,
dynamic_params: {
"openmeter_api_key": "password",
"openmeter_base_url": "text"
},
[Callbacks.LangSmith]: {
logo: `${asset_logos_folder}langsmith.png`,
supports_key_team_logging: true,
dynamic_params: {
"langsmith_api_key": "password",
"langsmith_project": "text",
"langsmith_base_url": "text",
"langsmith_sampling_rate": "number"
},
description: "Langsmith Logging Integration"
description: "OpenMeter Logging Integration"
},
{
id: "otel",
displayName: "Open Telemetry",
logo: `${asset_logos_folder}otel.png`,
supports_key_team_logging: false,
dynamic_params: {
"otel_endpoint": "text",
"otel_headers": "text"
},
[Callbacks.Braintrust]: {
logo: `${asset_logos_folder}braintrust.png`,
supports_key_team_logging: false,
dynamic_params: {},
description: "Braintrust Logging Integration"
description: "OpenTelemetry Logging Integration"
},
{
id: "s3",
displayName: "S3",
logo: `${asset_logos_folder}aws.svg`,
supports_key_team_logging: false,
dynamic_params: {
"s3_bucket_name": "text",
"aws_access_key_id": "password",
"aws_secret_access_key": "password",
"aws_region": "text"
},
[Callbacks.CustomCallbackAPI]: {
logo: `${asset_logos_folder}custom.svg`,
supports_key_team_logging: true,
dynamic_params: {},
description: "Custom Callback API Logging Integration"
},
[Callbacks.Datadog]: {
logo: `${asset_logos_folder}datadog.png`,
supports_key_team_logging: false,
dynamic_params: {},
description: "Datadog Logging Integration"
},
[Callbacks.Lago]: {
logo: `${asset_logos_folder}lago.svg`,
supports_key_team_logging: false,
dynamic_params: {},
description: "Lago Billing Logging Integration"
},
[Callbacks.OpenMeter]: {
logo: `${asset_logos_folder}openmeter.png`,
supports_key_team_logging: false,
dynamic_params: {},
description: "OpenMeter Logging Integration"
},
[Callbacks.OTel]: {
logo: `${asset_logos_folder}otel.png`,
supports_key_team_logging: false,
dynamic_params: {},
description: "OpenTelemetry Logging Integration"
},
[Callbacks.S3]: {
logo: `${asset_logos_folder}aws.svg`,
supports_key_team_logging: false,
dynamic_params: {},
description: "S3 Bucket (AWS) Logging Integration"
}
description: "S3 Bucket (AWS) Logging Integration"
}
];
// Utility functions for easy access
export const getCallbackById = (id: string): CallbackConfig | undefined => {
return CALLBACK_CONFIGS.find(callback => callback.id === id);
};
export const getCallbackByDisplayName = (displayName: string): CallbackConfig | undefined => {
return CALLBACK_CONFIGS.find(callback => callback.displayName === displayName);
};

View file

@ -45,9 +45,8 @@ import {
import AlertingSettings from "./alerting/alerting_settings";
import FormItem from "antd/es/form/FormItem";
import {
callback_map,
callbackInfo,
Callbacks,
CALLBACK_CONFIGS,
getCallbackById,
} from "./callback_info_helpers";
import { parseErrorMessage } from "./shared/errorUtils";
interface SettingsPageProps {
@ -239,18 +238,21 @@ const Settings: React.FC<SettingsPageProps> = ({
}
};
const handleSelectedCallbackChange = (
callbackObject: genericCallbackParams
) => {
setSelectedCallback(callbackObject.litellm_callback_name);
if (callbackObject && callbackObject.litellm_callback_params) {
setSelectedCallbackParams(callbackObject.litellm_callback_params);
const handleSelectedCallbackChange = (callbackName: string) => {
setSelectedCallback(callbackName);
// Get the callback configuration using the new clean structure
const callbackConfig = getCallbackById(callbackName);
// Get the parameters from the callback configuration
if (callbackConfig?.dynamic_params) {
const params = Object.keys(callbackConfig.dynamic_params);
setSelectedCallbackParams(params);
} else {
setSelectedCallbackParams([]);
}
};
const handleSaveAlerts = async () => {
if (!accessToken) {
return;
@ -639,74 +641,124 @@ const Settings: React.FC<SettingsPageProps> = ({
wrapperCol={{ span: 16 }}
labelAlign="left"
>
<>
<FormItem
label="Callback"
name="callback"
rules={[{ required: true, message: "Please select a callback" }]}
>
<Select
placeholder="Choose a logging callback..."
size="large"
className="w-full"
showSearch
filterOption={(input, option) =>
(option?.children?.toString() ?? "")
.toLowerCase()
.includes(input.toLowerCase())
}
onChange={(value) => {
const selectedCallback = allCallbacks[value];
if (selectedCallback) {
handleSelectedCallbackChange(selectedCallback);
}
handleSelectedCallbackChange(value);
}}
>
{Object.entries(Callbacks).map(
([callbackEnum, callbackDisplayName]) => (
<SelectItem
key={callbackDisplayName}
value={callback_map[callbackEnum]}
>
<div className="flex items-center space-x-2">
{callbackInfo[callbackDisplayName]?.logo ? (
<div className="w-5 h-5 flex items-center justify-center">
<img
src={callbackInfo[callbackDisplayName].logo}
alt={`${callbackEnum} logo`}
className="w-5 h-5"
onError={(e) => {
e.currentTarget.style.display = 'none';
}}
/>
</div>
) : (
<div className="w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs">
{(callbackDisplayName as string)
.charAt(0)
.toUpperCase()}
</div>
)}
<span>{callbackDisplayName}</span>
{CALLBACK_CONFIGS.map((callbackConfig) => (
<SelectItem
key={callbackConfig.id}
value={callbackConfig.id}
>
<div className="flex items-center space-x-3 py-1">
<div className="w-6 h-6 flex items-center justify-center">
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
src={callbackConfig.logo}
alt={`${callbackConfig.displayName} logo`}
className="w-6 h-6 rounded object-contain"
onError={(e) => {
e.currentTarget.style.display = 'none';
}}
/>
</div>
</SelectItem>
)
)}
<span className="font-medium text-gray-900">
{callbackConfig.displayName}
</span>
</div>
</SelectItem>
))}
</Select>
</FormItem>
{selectedCallbackParams &&
selectedCallbackParams.map((param) => (
<FormItem
label={param}
name={param}
key={param}
rules={[
{
required: true,
message: "Please enter the value for " + param,
},
]}
>
<Input.Password />
</FormItem>
))}
{selectedCallbackParams && selectedCallbackParams.length > 0 && (
<div className="space-y-4 mt-6 p-4 bg-gray-50 rounded-lg border">
{selectedCallbackParams.map((param) => {
// Get the callback configuration to look up parameter types
const callbackConfig = getCallbackById(selectedCallback || '');
const paramType = callbackConfig?.dynamic_params[param] || "text";
const fieldLabel = param.replace(/_/g, " ").replace(/\b\w/g, l => l.toUpperCase());
return (
<FormItem
label={
<span className="text-sm font-medium text-gray-700">
{fieldLabel}
<span className="text-red-500 ml-1">*</span>
</span>
}
name={param}
key={param}
className="mb-4"
rules={[
{
required: true,
message: `Please enter the ${fieldLabel.toLowerCase()}`,
},
]}
>
{paramType === "password" ? (
<Input.Password
size="large"
placeholder={`Enter your ${fieldLabel.toLowerCase()}`}
className="w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500"
/>
) : paramType === "number" ? (
<Input
type="number"
size="large"
placeholder={`Enter ${fieldLabel.toLowerCase()}`}
className="w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500"
min={0}
max={1}
step={0.1}
/>
) : (
<Input
size="large"
placeholder={`Enter your ${fieldLabel.toLowerCase()}`}
className="w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500"
/>
)}
</FormItem>
);
})}
</div>
)}
<div style={{ textAlign: "right", marginTop: "10px" }}>
<Button2 htmlType="submit">Save</Button2>
<div className="flex justify-end space-x-3 pt-6 mt-6 border-t border-gray-200">
<Button
onClick={() => {
setShowAddCallbacksModal(false);
setSelectedCallback(null);
setSelectedCallbackParams([]);
addForm.resetFields();
}}
>
Cancel
</Button>
<Button2
htmlType="submit"
>
Add Callback
</Button2>
</div>
</>
</Form>
</Modal>