fix(proxy): filter runtime callback inventory by object identity and label read-only rows in the UI

Runtime-only rows were filtered by callback name, which missed initialized
CustomLogger instances, router and proxy hook methods, guardrails, and
user functions. The inventory now inspects the live callback objects
through a public LoggingCallbackManager.get_callback_objects accessor
and hides litellm-internal hooks, guardrails, and instances of already
configured callbacks. The dashboard shows a Read only label for
runtime-only rows instead of an empty action cell

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-09 06:50:30 +00:00
parent 7c5660e886
commit f2558b4e5b
6 changed files with 125 additions and 197 deletions

View file

@ -441,6 +441,14 @@ class LoggingCallbackManager:
return result
def get_callback_objects(self) -> tuple[tuple[str, CustomLogger | Callable], ...]:
"""Every non-string active callback paired with the name `get_callbacks_by_type` reports it under."""
return tuple(
(self._get_callback_string(callback), callback)
for callback in self._get_all_callbacks()
if not isinstance(callback, str)
)
def _get_callback_string(self, callback: CustomLogger | Callable | str) -> str:
from litellm.litellm_core_utils.custom_logger_registry import (
CustomLoggerRegistry,

View file

@ -17551,7 +17551,6 @@ async def delete_callback(
def _normalize_callback_alias(callback_name: str) -> str:
"""Return the canonical callback name used for display and deduplication."""
callback_aliases: Final = (
("opentelemetry", "otel"),
("s3_v2", "s3"),
@ -17564,60 +17563,40 @@ def _normalize_callback_alias(callback_name: str) -> str:
)
def _callback_display_name(callback: CustomLogger) -> str:
"""Return the name an active callback instance is displayed under."""
def _callback_module_name(callback: CustomLogger | Callable[..., object]) -> str:
if inspect.ismethod(callback):
return callback.__func__.__module__
if inspect.isfunction(callback):
return callback.__module__
return type(callback).__module__
def _is_litellm_internal_callback(callback: CustomLogger | Callable[..., object]) -> bool:
"""Hooks litellm registers on its own (router, proxy, service logging) are not user logging callbacks."""
from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry
return CustomLoggerRegistry.get_callback_str_from_class_type(type(callback)) or type(callback).__name__
module_owner: Final = _callback_module_name(callback).partition(".")[0]
is_registered_integration: Final = CustomLoggerRegistry.get_callback_str_from_class_type(type(callback)) is not None
return not is_registered_integration and module_owner in ("litellm", "litellm_enterprise")
def _hidden_runtime_callback_names(configured_callback_names: frozenset[str]) -> frozenset[str]:
"""Return runtime callback names that are guardrails or instances of an already configured callback."""
"""Return runtime callback names that are internal hooks, guardrails, or instances of a configured callback."""
from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry
guardrail_names: Final = frozenset(
_callback_display_name(guardrail)
for guardrail in litellm.logging_callback_manager.get_custom_loggers_for_type(CustomGuardrail)
configured_classes: Final = tuple(
CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE[name]
for name in configured_callback_names
if name in CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE
)
configured_instance_names: Final = frozenset(
_callback_display_name(instance)
for configured_name in configured_callback_names
if configured_name in CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE
for instance in litellm.logging_callback_manager.get_custom_loggers_for_type(
CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE[configured_name]
)
)
configured_modules: Final = frozenset(
configured_name.rsplit(".", 1)[0] for configured_name in configured_callback_names if "." in configured_name
)
dotted_instance_names: Final = frozenset(
_callback_display_name(instance)
for instance in litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger)
if type(instance).__module__ in configured_modules
)
return guardrail_names | configured_instance_names | dotted_instance_names
def _is_runtime_logging_callback(callback_name: str, hidden_callback_names: frozenset[str]) -> bool:
"""Return whether a runtime callback name belongs in the logging inventory."""
internal_callback_names: Final = frozenset(
(
"_ProxyDBLogger",
"async_deployment_callback_on_failure",
"cache",
"deployment_callback_on_failure",
"deployment_callback_on_success",
"ResponsesIDSecurity",
"ServiceLogging",
"ShadowEvalLogger",
"SkillsInjectionHook",
"sync_deployment_callback_on_success",
)
)
return (
not callback_name.startswith("_PROXY_")
and callback_name not in internal_callback_names
and callback_name not in hidden_callback_names
configured_modules: Final = frozenset(name.rsplit(".", 1)[0] for name in configured_callback_names if "." in name)
return frozenset(
callback_name
for callback_name, callback in litellm.logging_callback_manager.get_callback_objects()
if isinstance(callback, CustomGuardrail)
or _is_litellm_internal_callback(callback)
or isinstance(callback, configured_classes)
or _callback_module_name(callback) in configured_modules
)
@ -17691,48 +17670,26 @@ async def get_config(
_normalize_callback_alias(callback)
for callback in (_success_callbacks + _failure_callbacks + _success_and_failure_callbacks)
)
runtime_callback_types: Final = (
("success", "success"),
("failure", "failure"),
("success_and_failure", "success_and_failure"),
)
runtime_callbacks_by_type: Final = litellm.logging_callback_manager.get_callbacks_by_type()
hidden_callback_names: Final = _hidden_runtime_callback_names(configured_callback_names)
runtime_callbacks: Final = tuple(
(callback, callback_mode)
for callback_type, callback_mode in runtime_callback_types
for callback in runtime_callbacks_by_type.get(callback_type, ())
if _is_runtime_logging_callback(callback, hidden_callback_names)
)
runtime_callback_rows: Final = tuple(
(
_normalize_callback_alias(callback),
callback_mode,
(_normalize_callback_alias(callback_name), callback_type)
for callback_type, callback_names in (
("success", runtime_callbacks_by_type["success"]),
("failure", runtime_callbacks_by_type["failure"]),
("success_and_failure", runtime_callbacks_by_type["success_and_failure"]),
)
for callback, callback_mode in runtime_callbacks
for callback_name in callback_names
if callback_name not in hidden_callback_names
)
unique_runtime_callback_rows: Final = tuple(
sorted(
(callback_name, callback_mode)
for index, (callback_name, callback_mode) in enumerate(runtime_callback_rows)
if callback_name not in configured_callback_names
and (callback_name, callback_mode) not in runtime_callback_rows[:index]
)
runtime_only_rows: Final = sorted(
frozenset(row for row in runtime_callback_rows if row[0] not in configured_callback_names)
)
runtime_rows: Final = tuple(
dict(
process_callback(
callback_name,
callback_mode,
environment_variables,
),
read_only=True,
)
for callback_name, callback_mode in unique_runtime_callback_rows
_data_to_return.extend(
dict(process_callback(callback_name, callback_type, environment_variables), read_only=True)
for callback_name, callback_type in runtime_only_rows
)
_data_to_return.extend(runtime_rows)
_data_to_return = _apply_callback_role_gate(_data_to_return, is_full_admin)
# Check if slack alerting is on

View file

@ -1095,7 +1095,8 @@ def test_get_config_callbacks_redacts_email_alerting_vars_for_view_only_admin(
def test_get_config_callbacks_appends_runtime_only_callbacks(client, auth_as, mock_prisma, monkeypatch):
"""Runtime-registered callbacks (not in config) are appended as read_only rows."""
"""LIT-5281: a YAML callback that the DB callback list replaced in the merged config still runs, so it must
show up as a read_only row next to the editable DB-configured one."""
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
@ -1113,47 +1114,30 @@ def test_get_config_callbacks_appends_runtime_only_callbacks(client, auth_as, mo
)
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
# Mock runtime callbacks: register otel in addition to langfuse in config.
import litellm
from litellm.integrations.langsmith import LangsmithLogger
from litellm.integrations.opentelemetry import OpenTelemetry
monkeypatch.setattr(litellm, "callbacks", ["otel"])
monkeypatch.setattr(
litellm.logging_callback_manager,
"get_callbacks_by_type",
MagicMock(
return_value={
"success": [],
"failure": [],
"success_and_failure": ["otel"],
}
),
)
monkeypatch.setattr(litellm, "success_callback", ["langfuse", LangsmithLogger()])
monkeypatch.setattr(litellm, "_async_success_callback", [])
monkeypatch.setattr(litellm, "failure_callback", [])
monkeypatch.setattr(litellm, "_async_failure_callback", [])
monkeypatch.setattr(litellm, "callbacks", [OpenTelemetry()])
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.get("/get/config/callbacks")
assert response.status_code == 200
body = response.json()
callbacks = body["callbacks"]
callback_names = [cb["name"] for cb in callbacks]
# Both should be present
assert "langfuse" in callback_names
assert "otel" in callback_names
# Configured callback should NOT be marked read_only
langfuse_cb = next(cb for cb in callbacks if cb["name"] == "langfuse")
assert not langfuse_cb.get("read_only")
# Runtime-only callback should be marked read_only
otel_cb = next(cb for cb in callbacks if cb["name"] == "otel")
assert otel_cb["read_only"] is True
assert otel_cb["type"] == "success_and_failure"
assert {callback["name"] for callback in callbacks} == {"langfuse", "otel"}
assert [(cb["name"], cb["type"], cb.get("read_only", False)) for cb in response.json()["callbacks"]] == [
("langfuse", "success", False),
("langsmith", "success", True),
("otel", "success_and_failure", True),
]
def test_get_config_callbacks_deduplicates_configured_and_runtime(client, auth_as, mock_prisma, monkeypatch):
"""When same callback is in both config and runtime, show only once as configured."""
"""A configured callback shows once as editable, whether the runtime holds its string or an initialized instance
registered under a different alias (arize initializes an OpenTelemetry instance)."""
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
@ -1164,48 +1148,37 @@ def test_get_config_callbacks_deduplicates_configured_and_runtime(client, auth_a
fake_proxy_config = MagicMock()
fake_proxy_config.get_config = AsyncMock(
return_value={
"litellm_settings": {"success_callback": ["langfuse"]},
"litellm_settings": {"success_callback": ["langfuse", "arize"]},
"general_settings": {},
"environment_variables": dict(_CALLBACK_ENV_FIXTURE),
}
)
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
# Mock runtime: same callback registered that is also in config
import litellm
from litellm.integrations.opentelemetry import OpenTelemetry
monkeypatch.setattr(litellm, "success_callback", ["langfuse"])
monkeypatch.setattr(litellm, "success_callback", ["langfuse", OpenTelemetry()])
monkeypatch.setattr(litellm, "callbacks", [])
monkeypatch.setattr(litellm, "failure_callback", [])
monkeypatch.setattr(litellm, "_async_success_callback", [])
monkeypatch.setattr(litellm, "_async_failure_callback", [])
monkeypatch.setattr(
litellm.logging_callback_manager,
"get_callbacks_by_type",
MagicMock(
return_value={
"success": ["langfuse"],
"failure": [],
"success_and_failure": [],
}
),
)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.get("/get/config/callbacks")
assert response.status_code == 200
body = response.json()
callbacks = body["callbacks"]
langfuse_rows = [cb for cb in callbacks if cb["name"] == "langfuse"]
# Should appear exactly once, not duplicated
assert len(langfuse_rows) == 1
# And it should NOT be marked read_only (it's in config)
assert not langfuse_rows[0].get("read_only")
assert {callback["name"] for callback in callbacks} == {"langfuse"}
assert [(cb["name"], cb["type"], cb.get("read_only", False)) for cb in response.json()["callbacks"]] == [
("langfuse", "success", False),
("arize", "success", False),
]
def _dotted_path_test_function(*args, **kwargs):
pass
@pytest.mark.parametrize("handler_kind", ["instance", "function"])
@pytest.mark.parametrize(
"config_key,expected_type",
[
@ -1215,9 +1188,9 @@ def test_get_config_callbacks_deduplicates_configured_and_runtime(client, auth_a
],
)
def test_get_config_callbacks_deduplicates_dotted_path_callback(
client, auth_as, mock_prisma, monkeypatch, config_key, expected_type
client, auth_as, mock_prisma, monkeypatch, config_key, expected_type, handler_kind
):
"""A dotted-path callback stays a single editable row instead of duplicating under its class name."""
"""A dotted-path callback stays a single editable row instead of duplicating under its class or function name."""
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
@ -1231,7 +1204,7 @@ def test_get_config_callbacks_deduplicates_dotted_path_callback(
class _DottedPathTestHandler(CustomLogger):
pass
dotted_handler = _DottedPathTestHandler()
dotted_handler = _DottedPathTestHandler() if handler_kind == "instance" else _dotted_path_test_function
dotted_path = f"{__name__}.dotted_handler"
fake_proxy_config = MagicMock()
@ -1249,17 +1222,6 @@ def test_get_config_callbacks_deduplicates_dotted_path_callback(
monkeypatch.setattr(litellm, "failure_callback", [])
monkeypatch.setattr(litellm, "_async_success_callback", [])
monkeypatch.setattr(litellm, "_async_failure_callback", [])
monkeypatch.setattr(
litellm.logging_callback_manager,
"get_callbacks_by_type",
MagicMock(
return_value={
"success": [],
"failure": [],
"success_and_failure": ["_DottedPathTestHandler"],
}
),
)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.get("/get/config/callbacks")
@ -1326,43 +1288,50 @@ def test_get_config_callbacks_excludes_internal_runtime_callbacks(client, auth_a
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
import litellm
from litellm._service_logger import ServiceLogging
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.langsmith import LangsmithLogger
from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter
from litellm.router import Router
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
class _InventoryTestGuardrail(CustomGuardrail):
pass
class _UserCodeLogger(CustomLogger):
pass
def user_code_function(*args, **kwargs):
pass
router = Router(model_list=[])
monkeypatch.setattr(litellm, "success_callback", [LangsmithLogger(), router.sync_deployment_callback_on_success])
monkeypatch.setattr(litellm, "_async_success_callback", [router.deployment_callback_on_success])
monkeypatch.setattr(litellm, "failure_callback", [user_code_function])
monkeypatch.setattr(litellm, "_async_failure_callback", [router.async_deployment_callback_on_failure])
monkeypatch.setattr(
litellm.logging_callback_manager,
"get_callbacks_by_type",
MagicMock(
return_value={
"success": ["langsmith", "deployment_callback_on_success", "cache"],
"failure": ["deployment_callback_on_failure"],
"success_and_failure": ["_ProxyDBLogger", "SkillsInjectionHook", "_InventoryTestGuardrail"],
}
),
)
monkeypatch.setattr(
litellm.logging_callback_manager,
"get_custom_loggers_for_type",
MagicMock(return_value=[_InventoryTestGuardrail(guardrail_name="inventory-test-guardrail")]),
litellm,
"callbacks",
[
_PROXY_MaxBudgetLimiter(),
_PROXY_LiteLLMManagedFiles(internal_usage_cache=MagicMock(), prisma_client=MagicMock()),
ServiceLogging(),
_InventoryTestGuardrail(guardrail_name="inventory-test-guardrail"),
_UserCodeLogger(),
],
)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.get("/get/config/callbacks")
assert response.status_code == 200
assert response.json()["callbacks"] == [
{
"name": "langsmith",
"variables": {
"LANGSMITH_API_KEY": None,
"LANGSMITH_PROJECT": None,
"LANGSMITH_DEFAULT_RUN_NAME": None,
},
"type": "success",
"read_only": True,
},
assert [
(callback["name"], callback["type"], callback["read_only"]) for callback in response.json()["callbacks"]
] == [
("_UserCodeLogger", "success_and_failure", True),
("langsmith", "success", True),
("user_code_function", "failure", True),
]
@ -1387,23 +1356,14 @@ def test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_adm
)
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
# Mock runtime: register otel
import litellm
monkeypatch.setattr(litellm, "success_callback", [])
monkeypatch.setattr(litellm, "_async_success_callback", [])
monkeypatch.setattr(litellm, "failure_callback", [])
monkeypatch.setattr(litellm, "_async_failure_callback", [])
monkeypatch.setattr(litellm, "callbacks", ["otel"])
monkeypatch.setattr(
litellm.logging_callback_manager,
"get_callbacks_by_type",
MagicMock(
return_value={
"success": [],
"failure": [],
"success_and_failure": ["otel"],
}
),
)
# View-only admin
with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY):
response = client.get("/get/config/callbacks")
assert response.status_code == 200
@ -1412,13 +1372,11 @@ def test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_adm
callbacks = body["callbacks"]
otel_cb = next((cb for cb in callbacks if cb["name"] == "otel"), None)
assert otel_cb is not None
# Secret env vars must be redacted
assert otel_cb["type"] == "success_and_failure"
assert otel_cb["read_only"] is True
assert otel_cb["variables"]["OTEL_HEADERS"] == "REDACTED"
# Non-secret vars should pass through
assert otel_cb["variables"]["OTEL_ENDPOINT"] == _CALLBACK_ENV_FIXTURE["OTEL_ENDPOINT"]
# Full admin sees secrets
with auth_as(LitellmUserRoles.PROXY_ADMIN):
admin_response = client.get("/get/config/callbacks")
assert admin_response.status_code == 200

View file

@ -97,7 +97,7 @@ describe("LoggingCallbacksTable", () => {
expect(onDelete).toHaveBeenCalledWith(callback);
});
it("hides the actions menu for read-only runtime callback rows", () => {
it("shows a read-only label instead of the actions menu for runtime-only callback rows", () => {
render(
<LoggingCallbacksTable
callbacks={[
@ -112,6 +112,7 @@ describe("LoggingCallbacksTable", () => {
);
expect(screen.getByTestId("callback-actions-langfuse-success")).toBeInTheDocument();
expect(screen.queryByTestId("callback-actions-datadog-success")).not.toBeInTheDocument();
expect(screen.getAllByText("Read only")).toHaveLength(1);
});
// Regression: `/get_callbacks` returns the same `name` twice when a

View file

@ -50,9 +50,15 @@ interface CallbackRowActionsProps {
}
function CallbackRowActions({ callback, onTest, onEdit, onDelete }: CallbackRowActionsProps) {
// Hide actions for read-only (runtime-only) callbacks.
if (callback.read_only) {
return null;
return (
<span
className="text-xs text-muted-foreground"
title="Active callback that was not added through the dashboard. Edit it where it was configured."
>
Read only
</span>
);
}
return (
<DropdownMenu>

View file

@ -8,8 +8,6 @@ export interface AlertingObject {
// every row to render as "Success".
type?: "success" | "failure" | "success_and_failure";
variables: AlertingVariables;
// read_only is true for runtime-only callbacks (not in DB/YAML config).
// UI hides edit/delete/test controls for read-only rows.
read_only?: boolean;
}