mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
fix(proxy): surface runtime-registered callbacks in /get/config/callbacks
Config-file callbacks fire at runtime but never appear in the UI Logging and Alerts page because /get/config/callbacks only reads the DB-merged config. Append runtime-registered callbacks from LoggingCallbackManager as read-only rows, deduplicated against configured rows via alias normalization. UI hides edit/delete/test actions for read-only rows.
This commit is contained in:
Yucheng Zhu
parent
36ea28b092
commit
8382341290
4 changed files with 208 additions and 0 deletions
|
|
@ -16992,6 +16992,22 @@ async def delete_callback(
|
|||
)
|
||||
|
||||
|
||||
def _normalize_callback_alias(callback_name: str) -> str:
|
||||
"""
|
||||
Normalize callback name aliases to their canonical form for deduplication.
|
||||
Examples: opentelemetry → otel, s3_v2 → s3, aws_sqs → sqs, custom_callback_api → generic_api.
|
||||
"""
|
||||
if not isinstance(callback_name, str):
|
||||
return str(callback_name)
|
||||
_alias_map: Final[dict[str, str]] = {
|
||||
"opentelemetry": "otel",
|
||||
"s3_v2": "s3",
|
||||
"aws_sqs": "sqs",
|
||||
"custom_callback_api": "generic_api",
|
||||
}
|
||||
return _alias_map.get(callback_name, callback_name)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/get/config/callbacks",
|
||||
tags=["config.yaml"],
|
||||
|
|
@ -17058,6 +17074,41 @@ async def get_config(
|
|||
for _callback in _success_and_failure_callbacks:
|
||||
_data_to_return.append(process_callback(_callback, "success_and_failure", environment_variables))
|
||||
|
||||
# Append runtime-only callbacks (registered but not in config).
|
||||
# Build a set of configured callback names (normalized for alias matching).
|
||||
_configured_callback_names_normalized: Final[set] = set()
|
||||
for _cb in _success_callbacks + _failure_callbacks + _success_and_failure_callbacks:
|
||||
_normalized = _normalize_callback_alias(_cb)
|
||||
_configured_callback_names_normalized.add(_normalized)
|
||||
|
||||
# Collect runtime-registered callbacks from LoggingCallbackManager.
|
||||
try:
|
||||
_runtime_callbacks_by_type = litellm.logging_callback_manager.get_callbacks_by_type()
|
||||
# Flatten all runtime callbacks with their types.
|
||||
_runtime_items: Final[list[tuple[str, str]]] = []
|
||||
for _cb_name in _runtime_callbacks_by_type.get("success", []):
|
||||
_runtime_items.append((_cb_name, "success"))
|
||||
for _cb_name in _runtime_callbacks_by_type.get("failure", []):
|
||||
_runtime_items.append((_cb_name, "failure"))
|
||||
for _cb_name in _runtime_callbacks_by_type.get("success_and_failure", []):
|
||||
_runtime_items.append((_cb_name, "success_and_failure"))
|
||||
|
||||
# Track normalized names of rows already added to avoid duplicates.
|
||||
_added_normalized_names: Final[set] = set(_configured_callback_names_normalized)
|
||||
|
||||
# Append runtime-only rows (those not in config).
|
||||
for _runtime_cb_name, _runtime_cb_type in _runtime_items:
|
||||
_normalized_runtime = _normalize_callback_alias(_runtime_cb_name)
|
||||
# Skip if this callback is in config or already appended.
|
||||
if _normalized_runtime not in _added_normalized_names:
|
||||
_added_normalized_names.add(_normalized_runtime)
|
||||
_runtime_row = process_callback(_runtime_cb_name, _runtime_cb_type, environment_variables)
|
||||
_runtime_row["read_only"] = True
|
||||
_data_to_return.append(_runtime_row)
|
||||
except Exception as _e:
|
||||
# If runtime callback discovery fails, log but don't block the response.
|
||||
verbose_proxy_logger.warning("Failed to append runtime callbacks to get_config response: %s", _e)
|
||||
|
||||
_data_to_return = _apply_callback_role_gate(_data_to_return, is_full_admin)
|
||||
|
||||
# Check if slack alerting is on
|
||||
|
|
|
|||
|
|
@ -1035,6 +1035,156 @@ def test_get_config_callbacks_redacts_email_alerting_vars_for_view_only_admin(
|
|||
assert admin_email["SMTP_HOST"] == "smtp.resend.com"
|
||||
|
||||
|
||||
def test_get_config_callbacks_appends_runtime_only_callbacks(
|
||||
client, auth_as, mock_prisma, monkeypatch
|
||||
):
|
||||
"""Runtime-registered callbacks (not in config) are appended as read_only rows."""
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
_install_litellm_config(mock_prisma)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.setattr(ps, "llm_router", None)
|
||||
|
||||
fake_proxy_config = MagicMock()
|
||||
fake_proxy_config.get_config = AsyncMock(
|
||||
return_value={
|
||||
"litellm_settings": {"success_callback": ["langfuse"]},
|
||||
"general_settings": {},
|
||||
"environment_variables": dict(_CALLBACK_ENV_FIXTURE),
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
|
||||
|
||||
# Mock runtime callbacks: register otel in addition to langfuse in config.
|
||||
import litellm
|
||||
|
||||
original = litellm.callbacks
|
||||
try:
|
||||
litellm.callbacks = ["otel"]
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
response = client.get("/get/config/callbacks")
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
|
||||
callbacks = body["callbacks"]
|
||||
callback_names = [cb["name"] for cb in callbacks]
|
||||
|
||||
# Both should be present
|
||||
assert "langfuse" in callback_names
|
||||
assert "otel" in callback_names
|
||||
|
||||
# Configured callback should NOT be marked read_only
|
||||
langfuse_cb = next(cb for cb in callbacks if cb["name"] == "langfuse")
|
||||
assert langfuse_cb.get("read_only") != True
|
||||
|
||||
# Runtime-only callback should be marked read_only
|
||||
otel_cb = next(cb for cb in callbacks if cb["name"] == "otel")
|
||||
assert otel_cb["read_only"] is True
|
||||
assert otel_cb["type"] == "success_and_failure"
|
||||
finally:
|
||||
litellm.callbacks = original
|
||||
|
||||
|
||||
def test_get_config_callbacks_deduplicates_configured_and_runtime(
|
||||
client, auth_as, mock_prisma, monkeypatch
|
||||
):
|
||||
"""When same callback is in both config and runtime, show only once as configured."""
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
_install_litellm_config(mock_prisma)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.setattr(ps, "llm_router", None)
|
||||
|
||||
fake_proxy_config = MagicMock()
|
||||
fake_proxy_config.get_config = AsyncMock(
|
||||
return_value={
|
||||
"litellm_settings": {"success_callback": ["langfuse"]},
|
||||
"general_settings": {},
|
||||
"environment_variables": dict(_CALLBACK_ENV_FIXTURE),
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
|
||||
|
||||
# Mock runtime: same callback registered that is also in config
|
||||
import litellm
|
||||
|
||||
original = litellm.success_callback
|
||||
try:
|
||||
litellm.success_callback = ["langfuse"]
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
response = client.get("/get/config/callbacks")
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
|
||||
callbacks = body["callbacks"]
|
||||
langfuse_rows = [cb for cb in callbacks if cb["name"] == "langfuse"]
|
||||
|
||||
# Should appear exactly once, not duplicated
|
||||
assert len(langfuse_rows) == 1
|
||||
# And it should NOT be marked read_only (it's in config)
|
||||
assert langfuse_rows[0].get("read_only") != True
|
||||
finally:
|
||||
litellm.success_callback = original
|
||||
|
||||
|
||||
def test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_admin(
|
||||
client, auth_as, mock_prisma, monkeypatch
|
||||
):
|
||||
"""Runtime-only callback rows are subject to the same redaction gate as configured."""
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
_install_litellm_config(mock_prisma)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.setattr(ps, "llm_router", None)
|
||||
|
||||
fake_proxy_config = MagicMock()
|
||||
fake_proxy_config.get_config = AsyncMock(
|
||||
return_value={
|
||||
"litellm_settings": {"success_callback": []},
|
||||
"general_settings": {},
|
||||
"environment_variables": dict(_CALLBACK_ENV_FIXTURE),
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
|
||||
|
||||
# Mock runtime: register otel
|
||||
import litellm
|
||||
|
||||
original = litellm.callbacks
|
||||
try:
|
||||
litellm.callbacks = ["otel"]
|
||||
# View-only admin
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY):
|
||||
response = client.get("/get/config/callbacks")
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
|
||||
callbacks = body["callbacks"]
|
||||
otel_cb = next((cb for cb in callbacks if cb["name"] == "otel"), None)
|
||||
assert otel_cb is not None
|
||||
|
||||
# Secret env vars must be redacted
|
||||
assert otel_cb["variables"]["OTEL_HEADERS"] == "REDACTED"
|
||||
# Non-secret vars should pass through
|
||||
assert otel_cb["variables"]["OTEL_ENDPOINT"] == _CALLBACK_ENV_FIXTURE["OTEL_ENDPOINT"]
|
||||
|
||||
# Full admin sees secrets
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
admin_response = client.get("/get/config/callbacks")
|
||||
assert admin_response.status_code == 200
|
||||
admin_body = admin_response.json()
|
||||
admin_otel = next(
|
||||
(cb for cb in admin_body["callbacks"] if cb["name"] == "otel"), None
|
||||
)
|
||||
assert admin_otel is not None
|
||||
assert admin_otel["variables"]["OTEL_HEADERS"] == _CALLBACK_ENV_FIXTURE["OTEL_HEADERS"]
|
||||
finally:
|
||||
litellm.callbacks = original
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /config/yaml
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -50,6 +50,10 @@ interface CallbackRowActionsProps {
|
|||
}
|
||||
|
||||
function CallbackRowActions({ callback, onTest, onEdit, onDelete }: CallbackRowActionsProps) {
|
||||
// Hide actions for read-only (runtime-only) callbacks.
|
||||
if (callback.read_only) {
|
||||
return null;
|
||||
}
|
||||
return (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger
|
||||
|
|
|
|||
|
|
@ -8,6 +8,9 @@ export interface AlertingObject {
|
|||
// every row to render as "Success".
|
||||
type?: "success" | "failure" | "success_and_failure";
|
||||
variables: AlertingVariables;
|
||||
// read_only is true for runtime-only callbacks (not in DB/YAML config).
|
||||
// UI hides edit/delete/test controls for read-only rows.
|
||||
read_only?: boolean;
|
||||
}
|
||||
|
||||
export interface AlertingVariables {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue