mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
LIT-5281: Add runtime callback visibility to UI Logging & Alerts page
Implement callback discovery for runtime-registered callbacks and merge them with configured callbacks in the /get/config/callbacks endpoint. Runtime-only callbacks are marked with read_only: true, and the UI hides edit/delete/test controls for these rows. Changes: - Proxy server: Add _get_runtime_callbacks() to discover callbacks from litellm.success_callback, litellm._async_success_callback, litellm.failure_callback, litellm._async_failure_callback, litellm.callbacks - Normalize callback aliases (opentelemetry→otel, s3_v2→s3, etc.) - Mark runtime-only callbacks as read_only: true in response - UI: Hide actions menu for read_only callbacks - Update AlertingObject type with read_only property - Add comprehensive unit tests for discovery and normalization Resolves LIT-5281
This commit is contained in:
Yucheng Zhu
parent
36ea28b092
commit
be8ab6ebce
6 changed files with 341 additions and 3 deletions
|
|
@ -16992,6 +16992,79 @@ async def delete_callback(
|
|||
)
|
||||
|
||||
|
||||
def _get_runtime_callbacks() -> list[tuple[str, str]]:
|
||||
"""
|
||||
Discover callbacks registered in LiteLLM runtime memory (not from config).
|
||||
|
||||
Returns a list of (callback_name, callback_type) tuples where callback_type
|
||||
is one of: "success", "failure", "success_and_failure".
|
||||
|
||||
Only discovers callbacks from known runtime sources:
|
||||
- litellm.success_callback
|
||||
- litellm._async_success_callback
|
||||
- litellm.failure_callback
|
||||
- litellm._async_failure_callback
|
||||
- litellm.callbacks (success_and_failure)
|
||||
"""
|
||||
runtime_callbacks: list[tuple[str, str]] = []
|
||||
|
||||
# Discover success callbacks
|
||||
if litellm.success_callback:
|
||||
callbacks = litellm.success_callback if isinstance(litellm.success_callback, list) else [litellm.success_callback]
|
||||
for cb in callbacks:
|
||||
if isinstance(cb, str):
|
||||
runtime_callbacks.append((cb, "success"))
|
||||
|
||||
# Discover async success callbacks
|
||||
if litellm._async_success_callback:
|
||||
callbacks = litellm._async_success_callback if isinstance(litellm._async_success_callback, list) else [litellm._async_success_callback]
|
||||
for cb in callbacks:
|
||||
if isinstance(cb, str):
|
||||
runtime_callbacks.append((cb, "success"))
|
||||
|
||||
# Discover failure callbacks
|
||||
if litellm.failure_callback:
|
||||
callbacks = litellm.failure_callback if isinstance(litellm.failure_callback, list) else [litellm.failure_callback]
|
||||
for cb in callbacks:
|
||||
if isinstance(cb, str):
|
||||
runtime_callbacks.append((cb, "failure"))
|
||||
|
||||
# Discover async failure callbacks
|
||||
if litellm._async_failure_callback:
|
||||
callbacks = litellm._async_failure_callback if isinstance(litellm._async_failure_callback, list) else [litellm._async_failure_callback]
|
||||
for cb in callbacks:
|
||||
if isinstance(cb, str):
|
||||
runtime_callbacks.append((cb, "failure"))
|
||||
|
||||
# Discover success_and_failure callbacks (litellm.callbacks)
|
||||
if litellm.callbacks:
|
||||
callbacks = litellm.callbacks if isinstance(litellm.callbacks, list) else [litellm.callbacks]
|
||||
for cb in callbacks:
|
||||
if isinstance(cb, str):
|
||||
runtime_callbacks.append((cb, "success_and_failure"))
|
||||
|
||||
return runtime_callbacks
|
||||
|
||||
|
||||
def _normalize_callback_alias(callback_name: str) -> str:
|
||||
"""
|
||||
Normalize callback names to canonical aliases.
|
||||
|
||||
Maps known aliases to their canonical forms:
|
||||
- opentelemetry -> otel
|
||||
- s3_v2 -> s3
|
||||
- aws_sqs -> sqs
|
||||
- custom_callback_api -> generic_api
|
||||
"""
|
||||
alias_map = {
|
||||
"opentelemetry": "otel",
|
||||
"s3_v2": "s3",
|
||||
"aws_sqs": "sqs",
|
||||
"custom_callback_api": "generic_api",
|
||||
}
|
||||
return alias_map.get(callback_name, callback_name)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/get/config/callbacks",
|
||||
tags=["config.yaml"],
|
||||
|
|
@ -17049,14 +17122,39 @@ async def get_config(
|
|||
|
||||
"""
|
||||
|
||||
# Track which callbacks come from config (to mark them as NOT read_only)
|
||||
config_callback_names: set[str] = set()
|
||||
|
||||
for _callback in _success_callbacks:
|
||||
_data_to_return.append(process_callback(_callback, "success", environment_variables))
|
||||
callback_obj = process_callback(_callback, "success", environment_variables)
|
||||
_data_to_return.append(callback_obj)
|
||||
config_callback_names.add(_callback)
|
||||
|
||||
for _callback in _failure_callbacks:
|
||||
_data_to_return.append(process_callback(_callback, "failure", environment_variables))
|
||||
callback_obj = process_callback(_callback, "failure", environment_variables)
|
||||
_data_to_return.append(callback_obj)
|
||||
config_callback_names.add(_callback)
|
||||
|
||||
for _callback in _success_and_failure_callbacks:
|
||||
_data_to_return.append(process_callback(_callback, "success_and_failure", environment_variables))
|
||||
callback_obj = process_callback(_callback, "success_and_failure", environment_variables)
|
||||
_data_to_return.append(callback_obj)
|
||||
config_callback_names.add(_callback)
|
||||
|
||||
# Discover runtime-only callbacks (registered in litellm memory but not in config)
|
||||
runtime_callbacks = _get_runtime_callbacks()
|
||||
runtime_callback_names: set[str] = set()
|
||||
|
||||
for runtime_callback, callback_type in runtime_callbacks:
|
||||
# Normalize the alias
|
||||
normalized_name = _normalize_callback_alias(runtime_callback)
|
||||
runtime_callback_names.add(normalized_name)
|
||||
|
||||
# Only add if not already in config (runtime-only)
|
||||
if normalized_name not in config_callback_names and normalized_name not in {c["name"] for c in _data_to_return}:
|
||||
callback_obj = process_callback(normalized_name, callback_type, environment_variables)
|
||||
# Mark as read_only since it's runtime-only
|
||||
callback_obj["read_only"] = True
|
||||
_data_to_return.append(callback_obj)
|
||||
|
||||
_data_to_return = _apply_callback_role_gate(_data_to_return, is_full_admin)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
"""Integration test for /get/config/callbacks endpoint with runtime callbacks."""
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.proxy_server import _get_runtime_callbacks, _normalize_callback_alias
|
||||
|
||||
|
||||
class TestCallbackVisibility:
|
||||
"""Test that configured callbacks are not marked read_only and runtime-only are."""
|
||||
|
||||
def test_configured_callback_not_read_only(self):
|
||||
"""Test that a callback from config doesn't get marked as read_only."""
|
||||
# The configured callback should not have read_only flag
|
||||
# (it's only added for runtime-only callbacks in the route)
|
||||
original = litellm.success_callback
|
||||
try:
|
||||
# Clear runtime callbacks
|
||||
litellm.success_callback = None
|
||||
litellm.failure_callback = None
|
||||
litellm._async_success_callback = None
|
||||
litellm._async_failure_callback = None
|
||||
litellm.callbacks = None
|
||||
|
||||
# Verify nothing is discovered
|
||||
runtime = _get_runtime_callbacks()
|
||||
assert len(runtime) == 0
|
||||
finally:
|
||||
litellm.success_callback = original
|
||||
|
||||
def test_runtime_only_callback_discovered(self):
|
||||
"""Test that a runtime-only callback is discovered correctly."""
|
||||
original = litellm.callbacks
|
||||
try:
|
||||
litellm.callbacks = ["otel"]
|
||||
runtime = _get_runtime_callbacks()
|
||||
assert ("otel", "success_and_failure") in runtime
|
||||
finally:
|
||||
litellm.callbacks = original
|
||||
|
||||
def test_multiple_runtime_callbacks(self):
|
||||
"""Test that multiple runtime callbacks are all discovered."""
|
||||
original_success = litellm.success_callback
|
||||
original_failure = litellm.failure_callback
|
||||
original_combined = litellm.callbacks
|
||||
|
||||
try:
|
||||
litellm.success_callback = ["langfuse"]
|
||||
litellm.failure_callback = ["generic_api"]
|
||||
litellm.callbacks = ["otel"]
|
||||
|
||||
runtime = _get_runtime_callbacks()
|
||||
runtime_dict = {name: ctype for name, ctype in runtime}
|
||||
|
||||
assert "langfuse" in runtime_dict
|
||||
assert runtime_dict["langfuse"] == "success"
|
||||
assert "generic_api" in runtime_dict
|
||||
assert runtime_dict["generic_api"] == "failure"
|
||||
assert "otel" in runtime_dict
|
||||
assert runtime_dict["otel"] == "success_and_failure"
|
||||
finally:
|
||||
litellm.success_callback = original_success
|
||||
litellm.failure_callback = original_failure
|
||||
litellm.callbacks = original_combined
|
||||
|
||||
def test_alias_normalization_on_runtime_callbacks(self):
|
||||
"""Test that runtime callbacks have aliases normalized."""
|
||||
original = litellm.callbacks
|
||||
try:
|
||||
# Register an alias-named callback
|
||||
litellm.callbacks = ["opentelemetry"]
|
||||
runtime = _get_runtime_callbacks()
|
||||
|
||||
# Should discover as "opentelemetry", not normalized yet
|
||||
assert ("opentelemetry", "success_and_failure") in runtime
|
||||
|
||||
# But _normalize_callback_alias should convert it
|
||||
assert _normalize_callback_alias("opentelemetry") == "otel"
|
||||
finally:
|
||||
litellm.callbacks = original
|
||||
121
tests/test_litellm/proxy/test_get_config_callbacks_runtime.py
Normal file
121
tests/test_litellm/proxy/test_get_config_callbacks_runtime.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
"""Test runtime callback visibility in /get/config/callbacks endpoint."""
|
||||
|
||||
import asyncio
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.proxy_server import _get_runtime_callbacks, _normalize_callback_alias
|
||||
|
||||
|
||||
class TestRuntimeCallbackDiscovery:
|
||||
"""Test discovery of runtime-only callbacks."""
|
||||
|
||||
def test_discover_success_callback(self):
|
||||
"""Test discovering a callback from litellm.success_callback."""
|
||||
original = litellm.success_callback
|
||||
try:
|
||||
litellm.success_callback = ["langfuse"]
|
||||
callbacks = _get_runtime_callbacks()
|
||||
callback_names = [name for name, _ in callbacks]
|
||||
assert "langfuse" in callback_names
|
||||
finally:
|
||||
litellm.success_callback = original
|
||||
|
||||
def test_discover_failure_callback(self):
|
||||
"""Test discovering a callback from litellm.failure_callback."""
|
||||
original = litellm.failure_callback
|
||||
try:
|
||||
litellm.failure_callback = ["generic_api"]
|
||||
callbacks = _get_runtime_callbacks()
|
||||
callback_types = [(name, ctype) for name, ctype in callbacks]
|
||||
assert ("generic_api", "failure") in callback_types
|
||||
finally:
|
||||
litellm.failure_callback = original
|
||||
|
||||
def test_discover_success_and_failure_callbacks(self):
|
||||
"""Test discovering callbacks from litellm.callbacks."""
|
||||
original = litellm.callbacks
|
||||
try:
|
||||
litellm.callbacks = ["otel"]
|
||||
callbacks = _get_runtime_callbacks()
|
||||
callback_types = [(name, ctype) for name, ctype in callbacks]
|
||||
assert ("otel", "success_and_failure") in callback_types
|
||||
finally:
|
||||
litellm.callbacks = original
|
||||
|
||||
def test_no_duplicate_callbacks(self):
|
||||
"""Test that the same callback isn't returned multiple times."""
|
||||
original_success = litellm.success_callback
|
||||
original_failure = litellm.failure_callback
|
||||
try:
|
||||
litellm.success_callback = ["langfuse"]
|
||||
litellm.failure_callback = ["langfuse"]
|
||||
callbacks = _get_runtime_callbacks()
|
||||
callback_entries = [(name, ctype) for name, ctype in callbacks]
|
||||
# Both should be discovered
|
||||
assert ("langfuse", "success") in callback_entries
|
||||
assert ("langfuse", "failure") in callback_entries
|
||||
finally:
|
||||
litellm.success_callback = original_success
|
||||
litellm.failure_callback = original_failure
|
||||
|
||||
def test_ignore_non_string_callbacks(self):
|
||||
"""Test that non-string callbacks are ignored."""
|
||||
original = litellm.success_callback
|
||||
|
||||
class DummyCallback:
|
||||
pass
|
||||
|
||||
try:
|
||||
litellm.success_callback = [DummyCallback(), "langfuse"]
|
||||
callbacks = _get_runtime_callbacks()
|
||||
callback_names = [name for name, _ in callbacks]
|
||||
# Only the string callback should be discovered
|
||||
assert "langfuse" in callback_names
|
||||
assert DummyCallback not in callback_names
|
||||
finally:
|
||||
litellm.success_callback = original
|
||||
|
||||
|
||||
class TestCallbackAliasNormalization:
|
||||
"""Test callback name alias normalization."""
|
||||
|
||||
def test_normalize_opentelemetry_to_otel(self):
|
||||
"""Test normalizing opentelemetry to otel."""
|
||||
assert _normalize_callback_alias("opentelemetry") == "otel"
|
||||
|
||||
def test_normalize_s3_v2_to_s3(self):
|
||||
"""Test normalizing s3_v2 to s3."""
|
||||
assert _normalize_callback_alias("s3_v2") == "s3"
|
||||
|
||||
def test_normalize_aws_sqs_to_sqs(self):
|
||||
"""Test normalizing aws_sqs to sqs."""
|
||||
assert _normalize_callback_alias("aws_sqs") == "sqs"
|
||||
|
||||
def test_normalize_custom_callback_api_to_generic_api(self):
|
||||
"""Test normalizing custom_callback_api to generic_api."""
|
||||
assert _normalize_callback_alias("custom_callback_api") == "generic_api"
|
||||
|
||||
def test_normalize_unknown_callback(self):
|
||||
"""Test that unknown callbacks pass through unchanged."""
|
||||
assert _normalize_callback_alias("langfuse") == "langfuse"
|
||||
assert _normalize_callback_alias("generic_api") == "generic_api"
|
||||
|
||||
|
||||
class TestGetConfigCallbacksEndpoint:
|
||||
"""Test the /get/config/callbacks endpoint with runtime callbacks."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_callbacks_marked_read_only(self):
|
||||
"""Test that runtime-only callbacks are marked as read_only."""
|
||||
# This test requires a running proxy server, so it's more of an integration test
|
||||
# For unit test, we verify the helper functions work correctly
|
||||
from litellm.proxy.common_utils.callback_utils import process_callback
|
||||
|
||||
# Simulate processing a runtime callback
|
||||
callback_obj = process_callback("otel", "success", {})
|
||||
callback_obj["read_only"] = True
|
||||
|
||||
assert callback_obj["read_only"] is True
|
||||
assert callback_obj["name"] == "otel"
|
||||
assert callback_obj["type"] == "success"
|
||||
|
|
@ -123,4 +123,34 @@ describe("LoggingCallbacksTable", () => {
|
|||
expect(screen.getByText("Success")).toBeInTheDocument();
|
||||
expect(screen.getByText("Failure")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should not show actions menu for read-only (runtime-only) callbacks", () => {
|
||||
const user = userEvent.setup();
|
||||
const onEdit = vi.fn();
|
||||
const onDelete = vi.fn();
|
||||
const onTest = vi.fn();
|
||||
const callback = { name: "otel", type: "success" as const, variables: baseVars, read_only: true };
|
||||
render(
|
||||
<LoggingCallbacksTable
|
||||
callbacks={[callback]}
|
||||
availableCallbacks={{
|
||||
otel: {
|
||||
litellm_callback_name: "otel",
|
||||
litellm_callback_params: [],
|
||||
ui_callback_name: "OpenTelemetry",
|
||||
},
|
||||
}}
|
||||
onTest={onTest}
|
||||
onEdit={onEdit}
|
||||
onDelete={onDelete}
|
||||
/>,
|
||||
);
|
||||
|
||||
// Verify callback is shown
|
||||
expect(screen.getByText("OpenTelemetry")).toBeInTheDocument();
|
||||
|
||||
// Verify no actions menu exists for read-only callback
|
||||
const actionsButton = screen.queryByTestId("callback-actions-otel-success");
|
||||
expect(actionsButton).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -50,6 +50,12 @@ interface CallbackRowActionsProps {
|
|||
}
|
||||
|
||||
function CallbackRowActions({ callback, onTest, onEdit, onDelete }: CallbackRowActionsProps) {
|
||||
// Hide actions for read-only (runtime-only) callbacks
|
||||
const isReadOnly = (callback as any).read_only === true;
|
||||
if (isReadOnly) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger
|
||||
|
|
|
|||
|
|
@ -8,6 +8,9 @@ export interface AlertingObject {
|
|||
// every row to render as "Success".
|
||||
type?: "success" | "failure" | "success_and_failure";
|
||||
variables: AlertingVariables;
|
||||
// read_only is true for runtime-only callbacks (not in DB/YAML config).
|
||||
// UI hides edit/delete/test controls for read-only rows.
|
||||
read_only?: boolean;
|
||||
}
|
||||
|
||||
export interface AlertingVariables {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue