From 8382341290cf99ff140d219d5e245a22c644d735 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Sat, 29 Aug 2026 17:55:44 -0700 Subject: [PATCH] fix(proxy): surface runtime-registered callbacks in /get/config/callbacks Config-file callbacks fire at runtime but never appear in the UI Logging and Alerts page because /get/config/callbacks only reads the DB-merged config. Append runtime-registered callbacks from LoggingCallbackManager as read-only rows, deduplicated against configured rows via alias normalization. UI hides edit/delete/test actions for read-only rows. --- litellm/proxy/proxy_server.py | 51 ++++++ .../proxy/proxy_server/test_routes_config.py | 150 ++++++++++++++++++ .../LoggingCallbacksTableColumns.tsx | 4 + .../LoggingCallbacks/types.ts | 3 + 4 files changed, 208 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 53a99065f2d..2420a47aa20 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16992,6 +16992,22 @@ async def delete_callback( ) +def _normalize_callback_alias(callback_name: str) -> str: + """ + Normalize callback name aliases to their canonical form for deduplication. + Examples: opentelemetry → otel, s3_v2 → s3, aws_sqs → sqs, custom_callback_api → generic_api. + """ + if not isinstance(callback_name, str): + return str(callback_name) + _alias_map: Final[dict[str, str]] = { + "opentelemetry": "otel", + "s3_v2": "s3", + "aws_sqs": "sqs", + "custom_callback_api": "generic_api", + } + return _alias_map.get(callback_name, callback_name) + + @router.get( "/get/config/callbacks", tags=["config.yaml"], @@ -17058,6 +17074,41 @@ async def get_config( for _callback in _success_and_failure_callbacks: _data_to_return.append(process_callback(_callback, "success_and_failure", environment_variables)) + # Append runtime-only callbacks (registered but not in config). + # Build a set of configured callback names (normalized for alias matching). + _configured_callback_names_normalized: Final[set] = set() + for _cb in _success_callbacks + _failure_callbacks + _success_and_failure_callbacks: + _normalized = _normalize_callback_alias(_cb) + _configured_callback_names_normalized.add(_normalized) + + # Collect runtime-registered callbacks from LoggingCallbackManager. + try: + _runtime_callbacks_by_type = litellm.logging_callback_manager.get_callbacks_by_type() + # Flatten all runtime callbacks with their types. + _runtime_items: Final[list[tuple[str, str]]] = [] + for _cb_name in _runtime_callbacks_by_type.get("success", []): + _runtime_items.append((_cb_name, "success")) + for _cb_name in _runtime_callbacks_by_type.get("failure", []): + _runtime_items.append((_cb_name, "failure")) + for _cb_name in _runtime_callbacks_by_type.get("success_and_failure", []): + _runtime_items.append((_cb_name, "success_and_failure")) + + # Track normalized names of rows already added to avoid duplicates. + _added_normalized_names: Final[set] = set(_configured_callback_names_normalized) + + # Append runtime-only rows (those not in config). + for _runtime_cb_name, _runtime_cb_type in _runtime_items: + _normalized_runtime = _normalize_callback_alias(_runtime_cb_name) + # Skip if this callback is in config or already appended. + if _normalized_runtime not in _added_normalized_names: + _added_normalized_names.add(_normalized_runtime) + _runtime_row = process_callback(_runtime_cb_name, _runtime_cb_type, environment_variables) + _runtime_row["read_only"] = True + _data_to_return.append(_runtime_row) + except Exception as _e: + # If runtime callback discovery fails, log but don't block the response. + verbose_proxy_logger.warning("Failed to append runtime callbacks to get_config response: %s", _e) + _data_to_return = _apply_callback_role_gate(_data_to_return, is_full_admin) # Check if slack alerting is on diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index ad3c470acf3..9b60b8e492d 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -1035,6 +1035,156 @@ def test_get_config_callbacks_redacts_email_alerting_vars_for_view_only_admin( assert admin_email["SMTP_HOST"] == "smtp.resend.com" +def test_get_config_callbacks_appends_runtime_only_callbacks( + client, auth_as, mock_prisma, monkeypatch +): + """Runtime-registered callbacks (not in config) are appended as read_only rows.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": ["langfuse"]}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + # Mock runtime callbacks: register otel in addition to langfuse in config. + import litellm + + original = litellm.callbacks + try: + litellm.callbacks = ["otel"] + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + body = response.json() + + callbacks = body["callbacks"] + callback_names = [cb["name"] for cb in callbacks] + + # Both should be present + assert "langfuse" in callback_names + assert "otel" in callback_names + + # Configured callback should NOT be marked read_only + langfuse_cb = next(cb for cb in callbacks if cb["name"] == "langfuse") + assert langfuse_cb.get("read_only") != True + + # Runtime-only callback should be marked read_only + otel_cb = next(cb for cb in callbacks if cb["name"] == "otel") + assert otel_cb["read_only"] is True + assert otel_cb["type"] == "success_and_failure" + finally: + litellm.callbacks = original + + +def test_get_config_callbacks_deduplicates_configured_and_runtime( + client, auth_as, mock_prisma, monkeypatch +): + """When same callback is in both config and runtime, show only once as configured.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": ["langfuse"]}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + # Mock runtime: same callback registered that is also in config + import litellm + + original = litellm.success_callback + try: + litellm.success_callback = ["langfuse"] + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + body = response.json() + + callbacks = body["callbacks"] + langfuse_rows = [cb for cb in callbacks if cb["name"] == "langfuse"] + + # Should appear exactly once, not duplicated + assert len(langfuse_rows) == 1 + # And it should NOT be marked read_only (it's in config) + assert langfuse_rows[0].get("read_only") != True + finally: + litellm.success_callback = original + + +def test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_admin( + client, auth_as, mock_prisma, monkeypatch +): + """Runtime-only callback rows are subject to the same redaction gate as configured.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": []}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + # Mock runtime: register otel + import litellm + + original = litellm.callbacks + try: + litellm.callbacks = ["otel"] + # View-only admin + with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + body = response.json() + + callbacks = body["callbacks"] + otel_cb = next((cb for cb in callbacks if cb["name"] == "otel"), None) + assert otel_cb is not None + + # Secret env vars must be redacted + assert otel_cb["variables"]["OTEL_HEADERS"] == "REDACTED" + # Non-secret vars should pass through + assert otel_cb["variables"]["OTEL_ENDPOINT"] == _CALLBACK_ENV_FIXTURE["OTEL_ENDPOINT"] + + # Full admin sees secrets + with auth_as(LitellmUserRoles.PROXY_ADMIN): + admin_response = client.get("/get/config/callbacks") + assert admin_response.status_code == 200 + admin_body = admin_response.json() + admin_otel = next( + (cb for cb in admin_body["callbacks"] if cb["name"] == "otel"), None + ) + assert admin_otel is not None + assert admin_otel["variables"]["OTEL_HEADERS"] == _CALLBACK_ENV_FIXTURE["OTEL_HEADERS"] + finally: + litellm.callbacks = original + + # --------------------------------------------------------------------------- # GET /config/yaml # --------------------------------------------------------------------------- diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx index 2263fe03b3d..97c554f7c49 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx @@ -50,6 +50,10 @@ interface CallbackRowActionsProps { } function CallbackRowActions({ callback, onTest, onEdit, onDelete }: CallbackRowActionsProps) { + // Hide actions for read-only (runtime-only) callbacks. + if (callback.read_only) { + return null; + } return (