diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 9b612993a69..8677440d0bb 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -441,6 +441,14 @@ class LoggingCallbackManager: return result + def get_callback_objects(self) -> tuple[tuple[str, CustomLogger | Callable], ...]: + """Every non-string active callback paired with the name `get_callbacks_by_type` reports it under.""" + return tuple( + (self._get_callback_string(callback), callback) + for callback in self._get_all_callbacks() + if not isinstance(callback, str) + ) + def _get_callback_string(self, callback: CustomLogger | Callable | str) -> str: from litellm.litellm_core_utils.custom_logger_registry import ( CustomLoggerRegistry, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1959fcee879..bdd399d46ce 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -17551,7 +17551,6 @@ async def delete_callback( def _normalize_callback_alias(callback_name: str) -> str: - """Return the canonical callback name used for display and deduplication.""" callback_aliases: Final = ( ("opentelemetry", "otel"), ("s3_v2", "s3"), @@ -17564,60 +17563,40 @@ def _normalize_callback_alias(callback_name: str) -> str: ) -def _callback_display_name(callback: CustomLogger) -> str: - """Return the name an active callback instance is displayed under.""" +def _callback_module_name(callback: CustomLogger | Callable[..., object]) -> str: + if inspect.ismethod(callback): + return callback.__func__.__module__ + if inspect.isfunction(callback): + return callback.__module__ + return type(callback).__module__ + + +def _is_litellm_internal_callback(callback: CustomLogger | Callable[..., object]) -> bool: + """Hooks litellm registers on its own (router, proxy, service logging) are not user logging callbacks.""" from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry - return CustomLoggerRegistry.get_callback_str_from_class_type(type(callback)) or type(callback).__name__ + module_owner: Final = _callback_module_name(callback).partition(".")[0] + is_registered_integration: Final = CustomLoggerRegistry.get_callback_str_from_class_type(type(callback)) is not None + return not is_registered_integration and module_owner in ("litellm", "litellm_enterprise") def _hidden_runtime_callback_names(configured_callback_names: frozenset[str]) -> frozenset[str]: - """Return runtime callback names that are guardrails or instances of an already configured callback.""" + """Return runtime callback names that are internal hooks, guardrails, or instances of a configured callback.""" from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry - guardrail_names: Final = frozenset( - _callback_display_name(guardrail) - for guardrail in litellm.logging_callback_manager.get_custom_loggers_for_type(CustomGuardrail) + configured_classes: Final = tuple( + CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE[name] + for name in configured_callback_names + if name in CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE ) - configured_instance_names: Final = frozenset( - _callback_display_name(instance) - for configured_name in configured_callback_names - if configured_name in CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE - for instance in litellm.logging_callback_manager.get_custom_loggers_for_type( - CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE[configured_name] - ) - ) - configured_modules: Final = frozenset( - configured_name.rsplit(".", 1)[0] for configured_name in configured_callback_names if "." in configured_name - ) - dotted_instance_names: Final = frozenset( - _callback_display_name(instance) - for instance in litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger) - if type(instance).__module__ in configured_modules - ) - return guardrail_names | configured_instance_names | dotted_instance_names - - -def _is_runtime_logging_callback(callback_name: str, hidden_callback_names: frozenset[str]) -> bool: - """Return whether a runtime callback name belongs in the logging inventory.""" - internal_callback_names: Final = frozenset( - ( - "_ProxyDBLogger", - "async_deployment_callback_on_failure", - "cache", - "deployment_callback_on_failure", - "deployment_callback_on_success", - "ResponsesIDSecurity", - "ServiceLogging", - "ShadowEvalLogger", - "SkillsInjectionHook", - "sync_deployment_callback_on_success", - ) - ) - return ( - not callback_name.startswith("_PROXY_") - and callback_name not in internal_callback_names - and callback_name not in hidden_callback_names + configured_modules: Final = frozenset(name.rsplit(".", 1)[0] for name in configured_callback_names if "." in name) + return frozenset( + callback_name + for callback_name, callback in litellm.logging_callback_manager.get_callback_objects() + if isinstance(callback, CustomGuardrail) + or _is_litellm_internal_callback(callback) + or isinstance(callback, configured_classes) + or _callback_module_name(callback) in configured_modules ) @@ -17691,48 +17670,26 @@ async def get_config( _normalize_callback_alias(callback) for callback in (_success_callbacks + _failure_callbacks + _success_and_failure_callbacks) ) - runtime_callback_types: Final = ( - ("success", "success"), - ("failure", "failure"), - ("success_and_failure", "success_and_failure"), - ) runtime_callbacks_by_type: Final = litellm.logging_callback_manager.get_callbacks_by_type() hidden_callback_names: Final = _hidden_runtime_callback_names(configured_callback_names) - runtime_callbacks: Final = tuple( - (callback, callback_mode) - for callback_type, callback_mode in runtime_callback_types - for callback in runtime_callbacks_by_type.get(callback_type, ()) - if _is_runtime_logging_callback(callback, hidden_callback_names) - ) runtime_callback_rows: Final = tuple( - ( - _normalize_callback_alias(callback), - callback_mode, + (_normalize_callback_alias(callback_name), callback_type) + for callback_type, callback_names in ( + ("success", runtime_callbacks_by_type["success"]), + ("failure", runtime_callbacks_by_type["failure"]), + ("success_and_failure", runtime_callbacks_by_type["success_and_failure"]), ) - for callback, callback_mode in runtime_callbacks + for callback_name in callback_names + if callback_name not in hidden_callback_names ) - unique_runtime_callback_rows: Final = tuple( - sorted( - (callback_name, callback_mode) - for index, (callback_name, callback_mode) in enumerate(runtime_callback_rows) - if callback_name not in configured_callback_names - and (callback_name, callback_mode) not in runtime_callback_rows[:index] - ) + runtime_only_rows: Final = sorted( + frozenset(row for row in runtime_callback_rows if row[0] not in configured_callback_names) ) - runtime_rows: Final = tuple( - dict( - process_callback( - callback_name, - callback_mode, - environment_variables, - ), - read_only=True, - ) - for callback_name, callback_mode in unique_runtime_callback_rows + _data_to_return.extend( + dict(process_callback(callback_name, callback_type, environment_variables), read_only=True) + for callback_name, callback_type in runtime_only_rows ) - _data_to_return.extend(runtime_rows) - _data_to_return = _apply_callback_role_gate(_data_to_return, is_full_admin) # Check if slack alerting is on diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index 99eab5b8611..2ae76dbda6f 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -1095,7 +1095,8 @@ def test_get_config_callbacks_redacts_email_alerting_vars_for_view_only_admin( def test_get_config_callbacks_appends_runtime_only_callbacks(client, auth_as, mock_prisma, monkeypatch): - """Runtime-registered callbacks (not in config) are appended as read_only rows.""" + """LIT-5281: a YAML callback that the DB callback list replaced in the merged config still runs, so it must + show up as a read_only row next to the editable DB-configured one.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -1113,47 +1114,30 @@ def test_get_config_callbacks_appends_runtime_only_callbacks(client, auth_as, mo ) monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) - # Mock runtime callbacks: register otel in addition to langfuse in config. import litellm + from litellm.integrations.langsmith import LangsmithLogger + from litellm.integrations.opentelemetry import OpenTelemetry - monkeypatch.setattr(litellm, "callbacks", ["otel"]) - monkeypatch.setattr( - litellm.logging_callback_manager, - "get_callbacks_by_type", - MagicMock( - return_value={ - "success": [], - "failure": [], - "success_and_failure": ["otel"], - } - ), - ) + monkeypatch.setattr(litellm, "success_callback", ["langfuse", LangsmithLogger()]) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "callbacks", [OpenTelemetry()]) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/get/config/callbacks") assert response.status_code == 200 - body = response.json() - callbacks = body["callbacks"] - callback_names = [cb["name"] for cb in callbacks] - - # Both should be present - assert "langfuse" in callback_names - assert "otel" in callback_names - - # Configured callback should NOT be marked read_only - langfuse_cb = next(cb for cb in callbacks if cb["name"] == "langfuse") - assert not langfuse_cb.get("read_only") - - # Runtime-only callback should be marked read_only - otel_cb = next(cb for cb in callbacks if cb["name"] == "otel") - assert otel_cb["read_only"] is True - assert otel_cb["type"] == "success_and_failure" - assert {callback["name"] for callback in callbacks} == {"langfuse", "otel"} + assert [(cb["name"], cb["type"], cb.get("read_only", False)) for cb in response.json()["callbacks"]] == [ + ("langfuse", "success", False), + ("langsmith", "success", True), + ("otel", "success_and_failure", True), + ] def test_get_config_callbacks_deduplicates_configured_and_runtime(client, auth_as, mock_prisma, monkeypatch): - """When same callback is in both config and runtime, show only once as configured.""" + """A configured callback shows once as editable, whether the runtime holds its string or an initialized instance + registered under a different alias (arize initializes an OpenTelemetry instance).""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -1164,48 +1148,37 @@ def test_get_config_callbacks_deduplicates_configured_and_runtime(client, auth_a fake_proxy_config = MagicMock() fake_proxy_config.get_config = AsyncMock( return_value={ - "litellm_settings": {"success_callback": ["langfuse"]}, + "litellm_settings": {"success_callback": ["langfuse", "arize"]}, "general_settings": {}, "environment_variables": dict(_CALLBACK_ENV_FIXTURE), } ) monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) - # Mock runtime: same callback registered that is also in config import litellm + from litellm.integrations.opentelemetry import OpenTelemetry - monkeypatch.setattr(litellm, "success_callback", ["langfuse"]) + monkeypatch.setattr(litellm, "success_callback", ["langfuse", OpenTelemetry()]) monkeypatch.setattr(litellm, "callbacks", []) monkeypatch.setattr(litellm, "failure_callback", []) monkeypatch.setattr(litellm, "_async_success_callback", []) monkeypatch.setattr(litellm, "_async_failure_callback", []) - monkeypatch.setattr( - litellm.logging_callback_manager, - "get_callbacks_by_type", - MagicMock( - return_value={ - "success": ["langfuse"], - "failure": [], - "success_and_failure": [], - } - ), - ) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/get/config/callbacks") assert response.status_code == 200 - body = response.json() - callbacks = body["callbacks"] - langfuse_rows = [cb for cb in callbacks if cb["name"] == "langfuse"] - - # Should appear exactly once, not duplicated - assert len(langfuse_rows) == 1 - # And it should NOT be marked read_only (it's in config) - assert not langfuse_rows[0].get("read_only") - assert {callback["name"] for callback in callbacks} == {"langfuse"} + assert [(cb["name"], cb["type"], cb.get("read_only", False)) for cb in response.json()["callbacks"]] == [ + ("langfuse", "success", False), + ("arize", "success", False), + ] +def _dotted_path_test_function(*args, **kwargs): + pass + + +@pytest.mark.parametrize("handler_kind", ["instance", "function"]) @pytest.mark.parametrize( "config_key,expected_type", [ @@ -1215,9 +1188,9 @@ def test_get_config_callbacks_deduplicates_configured_and_runtime(client, auth_a ], ) def test_get_config_callbacks_deduplicates_dotted_path_callback( - client, auth_as, mock_prisma, monkeypatch, config_key, expected_type + client, auth_as, mock_prisma, monkeypatch, config_key, expected_type, handler_kind ): - """A dotted-path callback stays a single editable row instead of duplicating under its class name.""" + """A dotted-path callback stays a single editable row instead of duplicating under its class or function name.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -1231,7 +1204,7 @@ def test_get_config_callbacks_deduplicates_dotted_path_callback( class _DottedPathTestHandler(CustomLogger): pass - dotted_handler = _DottedPathTestHandler() + dotted_handler = _DottedPathTestHandler() if handler_kind == "instance" else _dotted_path_test_function dotted_path = f"{__name__}.dotted_handler" fake_proxy_config = MagicMock() @@ -1249,17 +1222,6 @@ def test_get_config_callbacks_deduplicates_dotted_path_callback( monkeypatch.setattr(litellm, "failure_callback", []) monkeypatch.setattr(litellm, "_async_success_callback", []) monkeypatch.setattr(litellm, "_async_failure_callback", []) - monkeypatch.setattr( - litellm.logging_callback_manager, - "get_callbacks_by_type", - MagicMock( - return_value={ - "success": [], - "failure": [], - "success_and_failure": ["_DottedPathTestHandler"], - } - ), - ) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/get/config/callbacks") @@ -1326,43 +1288,50 @@ def test_get_config_callbacks_excludes_internal_runtime_callbacks(client, auth_a monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) import litellm + from litellm._service_logger import ServiceLogging from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.integrations.custom_logger import CustomLogger + from litellm.integrations.langsmith import LangsmithLogger + from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter + from litellm.router import Router + from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles class _InventoryTestGuardrail(CustomGuardrail): pass + class _UserCodeLogger(CustomLogger): + pass + + def user_code_function(*args, **kwargs): + pass + + router = Router(model_list=[]) + monkeypatch.setattr(litellm, "success_callback", [LangsmithLogger(), router.sync_deployment_callback_on_success]) + monkeypatch.setattr(litellm, "_async_success_callback", [router.deployment_callback_on_success]) + monkeypatch.setattr(litellm, "failure_callback", [user_code_function]) + monkeypatch.setattr(litellm, "_async_failure_callback", [router.async_deployment_callback_on_failure]) monkeypatch.setattr( - litellm.logging_callback_manager, - "get_callbacks_by_type", - MagicMock( - return_value={ - "success": ["langsmith", "deployment_callback_on_success", "cache"], - "failure": ["deployment_callback_on_failure"], - "success_and_failure": ["_ProxyDBLogger", "SkillsInjectionHook", "_InventoryTestGuardrail"], - } - ), - ) - monkeypatch.setattr( - litellm.logging_callback_manager, - "get_custom_loggers_for_type", - MagicMock(return_value=[_InventoryTestGuardrail(guardrail_name="inventory-test-guardrail")]), + litellm, + "callbacks", + [ + _PROXY_MaxBudgetLimiter(), + _PROXY_LiteLLMManagedFiles(internal_usage_cache=MagicMock(), prisma_client=MagicMock()), + ServiceLogging(), + _InventoryTestGuardrail(guardrail_name="inventory-test-guardrail"), + _UserCodeLogger(), + ], ) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/get/config/callbacks") assert response.status_code == 200 - assert response.json()["callbacks"] == [ - { - "name": "langsmith", - "variables": { - "LANGSMITH_API_KEY": None, - "LANGSMITH_PROJECT": None, - "LANGSMITH_DEFAULT_RUN_NAME": None, - }, - "type": "success", - "read_only": True, - }, + assert [ + (callback["name"], callback["type"], callback["read_only"]) for callback in response.json()["callbacks"] + ] == [ + ("_UserCodeLogger", "success_and_failure", True), + ("langsmith", "success", True), + ("user_code_function", "failure", True), ] @@ -1387,23 +1356,14 @@ def test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_adm ) monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) - # Mock runtime: register otel import litellm + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) monkeypatch.setattr(litellm, "callbacks", ["otel"]) - monkeypatch.setattr( - litellm.logging_callback_manager, - "get_callbacks_by_type", - MagicMock( - return_value={ - "success": [], - "failure": [], - "success_and_failure": ["otel"], - } - ), - ) - # View-only admin with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): response = client.get("/get/config/callbacks") assert response.status_code == 200 @@ -1412,13 +1372,11 @@ def test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_adm callbacks = body["callbacks"] otel_cb = next((cb for cb in callbacks if cb["name"] == "otel"), None) assert otel_cb is not None - - # Secret env vars must be redacted + assert otel_cb["type"] == "success_and_failure" + assert otel_cb["read_only"] is True assert otel_cb["variables"]["OTEL_HEADERS"] == "REDACTED" - # Non-secret vars should pass through assert otel_cb["variables"]["OTEL_ENDPOINT"] == _CALLBACK_ENV_FIXTURE["OTEL_ENDPOINT"] - # Full admin sees secrets with auth_as(LitellmUserRoles.PROXY_ADMIN): admin_response = client.get("/get/config/callbacks") assert admin_response.status_code == 200 diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx index 3b987a93478..a4d76e2cc06 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx @@ -97,7 +97,7 @@ describe("LoggingCallbacksTable", () => { expect(onDelete).toHaveBeenCalledWith(callback); }); - it("hides the actions menu for read-only runtime callback rows", () => { + it("shows a read-only label instead of the actions menu for runtime-only callback rows", () => { render( { ); expect(screen.getByTestId("callback-actions-langfuse-success")).toBeInTheDocument(); expect(screen.queryByTestId("callback-actions-datadog-success")).not.toBeInTheDocument(); + expect(screen.getAllByText("Read only")).toHaveLength(1); }); // Regression: `/get_callbacks` returns the same `name` twice when a diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx index 97c554f7c49..950ef5de6e4 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx @@ -50,9 +50,15 @@ interface CallbackRowActionsProps { } function CallbackRowActions({ callback, onTest, onEdit, onDelete }: CallbackRowActionsProps) { - // Hide actions for read-only (runtime-only) callbacks. if (callback.read_only) { - return null; + return ( + + Read only + + ); } return ( diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/types.ts b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/types.ts index 8d42d93bca7..ae446f1a042 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/types.ts +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/types.ts @@ -8,8 +8,6 @@ export interface AlertingObject { // every row to render as "Success". type?: "success" | "failure" | "success_and_failure"; variables: AlertingVariables; - // read_only is true for runtime-only callbacks (not in DB/YAML config). - // UI hides edit/delete/test controls for read-only rows. read_only?: boolean; }