diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b2e70cf66a8..efcd72873f1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -256,7 +256,7 @@ from litellm.constants import ( WEEKLY_SPEND_REPORT_JOB_ID, ) from litellm.exceptions import RejectedRequestError -from litellm.integrations.custom_guardrail import ModifyResponseException +from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.agentic_loop_settings import ( @@ -16993,19 +16993,66 @@ async def delete_callback( def _normalize_callback_alias(callback_name: str) -> str: - """ - Normalize callback name aliases to their canonical form for deduplication. - Examples: opentelemetry → otel, s3_v2 → s3, aws_sqs → sqs, custom_callback_api → generic_api. - """ - if not isinstance(callback_name, str): - return str(callback_name) - _alias_map: Final[dict[str, str]] = { - "opentelemetry": "otel", - "s3_v2": "s3", - "aws_sqs": "sqs", - "custom_callback_api": "generic_api", - } - return _alias_map.get(callback_name, callback_name) + """Return the canonical callback name used for display and deduplication.""" + callback_aliases: Final = ( + ("opentelemetry", "otel"), + ("s3_v2", "s3"), + ("aws_sqs", "sqs"), + ("custom_callback_api", "generic_api"), + ) + return next( + (canonical_name for alias, canonical_name in callback_aliases if alias == callback_name), + callback_name, + ) + + +def _callback_display_name(callback: CustomLogger) -> str: + """Return the name an active callback instance is displayed under.""" + from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry + + return CustomLoggerRegistry.get_callback_str_from_class_type(type(callback)) or type(callback).__name__ + + +def _hidden_runtime_callback_names(configured_callback_names: frozenset[str]) -> frozenset[str]: + """Return runtime callback names that are guardrails or instances of an already configured callback.""" + from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry + + guardrail_names: Final = frozenset( + _callback_display_name(guardrail) + for guardrail in litellm.logging_callback_manager.get_custom_loggers_for_type(CustomGuardrail) + ) + configured_instance_names: Final = frozenset( + _callback_display_name(instance) + for configured_name in configured_callback_names + if configured_name in CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE + for instance in litellm.logging_callback_manager.get_custom_loggers_for_type( + CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE[configured_name] + ) + ) + return guardrail_names | configured_instance_names + + +def _is_runtime_logging_callback(callback_name: str, hidden_callback_names: frozenset[str]) -> bool: + """Return whether a runtime callback name belongs in the logging inventory.""" + internal_callback_names: Final = frozenset( + ( + "_ProxyDBLogger", + "async_deployment_callback_on_failure", + "cache", + "deployment_callback_on_failure", + "deployment_callback_on_success", + "ResponsesIDSecurity", + "ServiceLogging", + "ShadowEvalLogger", + "SkillsInjectionHook", + "sync_deployment_callback_on_success", + ) + ) + return ( + not callback_name.startswith("_PROXY_") + and callback_name not in internal_callback_names + and callback_name not in hidden_callback_names + ) @router.get( @@ -17040,14 +17087,10 @@ async def get_config( # Normalize string callbacks to lists def normalize_callback(callback): if isinstance(callback, str): - return [callback] - elif callback is None: - return [] - elif isinstance(callback, list): - return callback - # Convert dict, tuple, set, or any other type to empty list - # (config validation should prevent non-list types, but guard here) - return [] + return (callback,) + if callback is None: + return () + return tuple(callback) if isinstance(callback, list) else () _success_callbacks = normalize_callback(_success_callbacks) _failure_callbacks = normalize_callback(_failure_callbacks) @@ -17078,55 +17121,51 @@ async def get_config( for _callback in _success_and_failure_callbacks: _data_to_return.append(process_callback(_callback, "success_and_failure", environment_variables)) - # Append runtime-only callbacks (registered but not in config). - # Build a set of configured callback names (normalized for alias matching). - _configured_callback_names_normalized: Final[set] = set() - for _cb in _success_callbacks + _failure_callbacks + _success_and_failure_callbacks: - _normalized = _normalize_callback_alias(_cb) - _configured_callback_names_normalized.add(_normalized) - - # Collect runtime-registered callbacks from LoggingCallbackManager. - try: - _runtime_callbacks_by_type = litellm.logging_callback_manager.get_callbacks_by_type() - # Flatten all runtime callbacks with their types. - _runtime_items: Final[list[tuple[str, str]]] = [] - for _cb_name in _runtime_callbacks_by_type.get("success", []): - _runtime_items.append((_cb_name, "success")) - for _cb_name in _runtime_callbacks_by_type.get("failure", []): - _runtime_items.append((_cb_name, "failure")) - for _cb_name in _runtime_callbacks_by_type.get("success_and_failure", []): - _runtime_items.append((_cb_name, "success_and_failure")) - - # Track normalized names of rows already added to avoid duplicates. - _added_normalized_names: Final[set] = set(_configured_callback_names_normalized) - - # Append runtime-only rows (those not in config). - # Filter out internal proxy hooks (names starting with _PROXY or known internal names). - _internal_callback_prefixes: Final[tuple] = ( - "_PROXY", - "_Async", - "ShadowEval", - "ServiceLogging", - "SkillsInjection", - "ResponsesID", + configured_callback_names: Final = frozenset( + _normalize_callback_alias(callback) + for callback in (_success_callbacks + _failure_callbacks + _success_and_failure_callbacks) + ) + runtime_callback_types: Final = ( + ("success", "success"), + ("failure", "failure"), + ("success_and_failure", "success_and_failure"), + ) + runtime_callbacks_by_type: Final = litellm.logging_callback_manager.get_callbacks_by_type() + hidden_callback_names: Final = _hidden_runtime_callback_names(configured_callback_names) + runtime_callbacks: Final = tuple( + (callback, callback_mode) + for callback_type, callback_mode in runtime_callback_types + for callback in runtime_callbacks_by_type.get(callback_type, ()) + if _is_runtime_logging_callback(callback, hidden_callback_names) + ) + runtime_callback_rows: Final = tuple( + ( + _normalize_callback_alias(callback), + callback_mode, ) - for _runtime_cb_name, _runtime_cb_type in _runtime_items: - # Skip internal proxy callbacks (these are infrastructure, not user-configured). - if isinstance(_runtime_cb_name, str) and any( - _runtime_cb_name.startswith(p) for p in _internal_callback_prefixes - ): - continue + for callback, callback_mode in runtime_callbacks + ) + unique_runtime_callback_rows: Final = tuple( + sorted( + (callback_name, callback_mode) + for index, (callback_name, callback_mode) in enumerate(runtime_callback_rows) + if callback_name not in configured_callback_names + and (callback_name, callback_mode) not in runtime_callback_rows[:index] + ) + ) + runtime_rows: Final = tuple( + dict( + process_callback( + callback_name, + callback_mode, + environment_variables, + ), + read_only=True, + ) + for callback_name, callback_mode in unique_runtime_callback_rows + ) - _normalized_runtime = _normalize_callback_alias(_runtime_cb_name) - # Skip if this callback is in config or already appended. - if _normalized_runtime not in _added_normalized_names: - _added_normalized_names.add(_normalized_runtime) - _runtime_row = process_callback(_runtime_cb_name, _runtime_cb_type, environment_variables) - _runtime_row["read_only"] = True - _data_to_return.append(_runtime_row) - except Exception as _e: - # If runtime callback discovery fails, log but don't block the response. - verbose_proxy_logger.warning("Failed to append runtime callbacks to get_config response: %s", _e) + _data_to_return.extend(runtime_rows) _data_to_return = _apply_callback_role_gate(_data_to_return, is_full_admin) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index 1242cf5f038..47ed4a01ea1 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -94,10 +94,7 @@ def test_config_update_no_db_error(client, auth_as, monkeypatch): json={"general_settings": {"alerting": ["slack"]}}, ) assert response.status_code != 200 - assert ( - "db" in str(response.json()).lower() - or "connect" in str(response.json()).lower() - ) + assert "db" in str(response.json()).lower() or "connect" in str(response.json()).lower() # --------------------------------------------------------------------------- @@ -138,9 +135,7 @@ def test_config_field_update_happy_admin(client, auth_as, mock_prisma, monkeypat } -def test_config_field_update_non_admin_rejected( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_update_non_admin_rejected(client, auth_as, mock_prisma, monkeypatch): """Non-admin cannot update config fields — returns 400 with not-allowed detail (handler uses 400 for the auth gate, not 403).""" from litellm.proxy import proxy_server as ps @@ -200,9 +195,7 @@ def test_config_field_info_happy_admin(client, auth_as, mock_prisma, monkeypatch monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get( - "/config/field/info", params={"field_name": "max_parallel_requests"} - ) + response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"}) assert response.status_code == 200 assert normalize(response.json()) == { "field_name": "max_parallel_requests", @@ -210,9 +203,7 @@ def test_config_field_info_happy_admin(client, auth_as, mock_prisma, monkeypatch } -def test_config_field_info_non_admin_rejected( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_info_non_admin_rejected(client, auth_as, mock_prisma, monkeypatch): """Non-admin (INTERNAL_USER) is denied — admin-view gate fires.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -221,9 +212,7 @@ def test_config_field_info_non_admin_rejected( monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.INTERNAL_USER): - response = client.get( - "/config/field/info", params={"field_name": "max_parallel_requests"} - ) + response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"}) assert response.status_code == 400 assert "error" in response.json().get("detail", {}) @@ -240,16 +229,12 @@ def test_config_field_info_field_not_in_db(client, auth_as, mock_prisma, monkeyp monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get( - "/config/field/info", params={"field_name": "max_parallel_requests"} - ) + response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"}) assert response.status_code == 400 assert "not in DB" in response.json().get("detail", {}).get("error", "") -def test_config_field_info_redacts_nested_secret_for_view_only_admin( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_info_redacts_nested_secret_for_view_only_admin(client, auth_as, mock_prisma, monkeypatch): """A view-only admin reading a structured field must not receive nested credentials. database_args carries aws_web_identity_token (a DynamoDB role-assumption credential); it must come back redacted while non-secret @@ -270,9 +255,7 @@ def test_config_field_info_redacts_nested_secret_for_view_only_admin( monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): - response = client.get( - "/config/field/info", params={"field_name": "database_args"} - ) + response = client.get("/config/field/info", params={"field_name": "database_args"}) assert response.status_code == 200 value = response.json()["field_value"] assert value["aws_web_identity_token"] == "REDACTED" @@ -280,9 +263,7 @@ def test_config_field_info_redacts_nested_secret_for_view_only_admin( assert value["user_table_name"] == "LiteLLM_UserTable" -def test_config_field_info_full_admin_sees_nested_secret( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_info_full_admin_sees_nested_secret(client, auth_as, mock_prisma, monkeypatch): """The redaction must not over-redact for a full PROXY_ADMIN, who needs the real nested value to populate the edit form.""" from litellm.proxy import proxy_server as ps @@ -300,18 +281,14 @@ def test_config_field_info_full_admin_sees_nested_secret( monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get( - "/config/field/info", params={"field_name": "database_args"} - ) + response = client.get("/config/field/info", params={"field_name": "database_args"}) assert response.status_code == 200 value = response.json()["field_value"] assert value["aws_web_identity_token"] == "sk-super-secret-token" assert value["region_name"] == "us-east-1" -def test_config_field_info_redacts_top_level_scalar_for_view_only( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_info_redacts_top_level_scalar_for_view_only(client, auth_as, mock_prisma, monkeypatch): """The top-level scalar branch must also redact for a view-only admin. database_url carries DB credentials and is not caught by the name masker, so it is in the explicit secret set.""" @@ -325,9 +302,7 @@ def test_config_field_info_redacts_top_level_scalar_for_view_only( monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): - response = client.get( - "/config/field/info", params={"field_name": "database_url"} - ) + response = client.get("/config/field/info", params={"field_name": "database_url"}) assert response.status_code == 200 assert response.json()["field_value"] == "REDACTED" @@ -341,17 +316,12 @@ def test_redact_general_setting_value_recurses_list_of_dicts(): {"path": "/foo", "headers": {"Authorization": "Bearer sk-x"}}, {"path": "/bar", "client_secret": "sk-y"}, ] - redacted = ps._redact_general_setting_value( - "some_list_field", value, is_full_admin=False - ) + redacted = ps._redact_general_setting_value("some_list_field", value, is_full_admin=False) assert redacted[0]["headers"]["Authorization"] == "REDACTED" assert redacted[0]["path"] == "/foo" assert redacted[1]["client_secret"] == "REDACTED" assert redacted[1]["path"] == "/bar" - assert ( - ps._redact_general_setting_value("some_list_field", value, is_full_admin=True) - == value - ) + assert ps._redact_general_setting_value("some_list_field", value, is_full_admin=True) == value def test_redact_secret_values_in_obj_fails_closed_at_max_depth(): @@ -369,22 +339,16 @@ def test_redact_secret_values_in_obj_fails_closed_at_max_depth(): for _ in range(ps._REDACT_SECRET_MAX_DEPTH + 2): nested = {"wrap": nested} - out = ps._redact_general_setting_value( - "some_struct_field", nested, is_full_admin=False - ) + out = ps._redact_general_setting_value("some_struct_field", nested, is_full_admin=False) # the secret must not survive anywhere in the returned tree assert "sk-leak-bottom" not in repr(out) # full admin is unaffected by the cap — the value comes back untouched - admin_out = ps._redact_general_setting_value( - "some_struct_field", nested, is_full_admin=True - ) + admin_out = ps._redact_general_setting_value("some_struct_field", nested, is_full_admin=True) assert admin_out is nested -def test_config_list_redacts_pass_through_secret_for_view_only( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_list_redacts_pass_through_secret_for_view_only(client, auth_as, mock_prisma, monkeypatch): """/config/list must not leak pass_through_endpoints upstream credentials to a view-only admin. pass_through_endpoints is a known secret-bearing field, so a non-admin gets it redacted; a full admin still sees it.""" @@ -411,24 +375,16 @@ def test_config_list_redacts_pass_through_secret_for_view_only( ) def _pass_through_value(body): - return next( - entry["field_value"] - for entry in body - if entry["field_name"] == "pass_through_endpoints" - ) + return next(entry["field_value"] for entry in body if entry["field_name"] == "pass_through_endpoints") with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): - view_resp = client.get( - "/config/list", params={"config_type": "general_settings"} - ) + view_resp = client.get("/config/list", params={"config_type": "general_settings"}) assert view_resp.status_code == 200 assert "sk-UPSTREAM-SECRET" not in view_resp.text assert _pass_through_value(view_resp.json()) == "REDACTED" with auth_as(LitellmUserRoles.PROXY_ADMIN): - admin_resp = client.get( - "/config/list", params={"config_type": "general_settings"} - ) + admin_resp = client.get("/config/list", params={"config_type": "general_settings"}) assert admin_resp.status_code == 200 admin_value = _pass_through_value(admin_resp.json()) assert admin_value[0]["headers"]["Authorization"] == "Bearer sk-UPSTREAM-SECRET" @@ -452,9 +408,7 @@ def test_config_list_happy_admin(client, auth_as, mock_prisma, monkeypatch): monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get( - "/config/list", params={"config_type": "general_settings"} - ) + response = client.get("/config/list", params={"config_type": "general_settings"}) assert response.status_code == 200 body = response.json() assert isinstance(body, list) @@ -560,9 +514,7 @@ def test_config_list_non_admin_rejected(client, auth_as, mock_prisma, monkeypatc monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.INTERNAL_USER): - response = client.get( - "/config/list", params={"config_type": "general_settings"} - ) + response = client.get("/config/list", params={"config_type": "general_settings"}) assert response.status_code == 400 assert "role" in response.json().get("detail", {}).get("error", "").lower() @@ -575,9 +527,7 @@ def test_config_list_no_db_error(client, auth_as, monkeypatch): monkeypatch.setattr(ps, "prisma_client", None) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get( - "/config/list", params={"config_type": "general_settings"} - ) + response = client.get("/config/list", params={"config_type": "general_settings"}) assert response.status_code == 400 assert "error" in response.json().get("detail", {}) @@ -621,9 +571,7 @@ def test_config_field_delete_happy_admin(client, auth_as, mock_prisma, monkeypat } -def test_config_field_delete_non_admin_rejected( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_delete_non_admin_rejected(client, auth_as, mock_prisma, monkeypatch): """Non-admin caller hits the 400 not-allowed branch with role in detail.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -643,9 +591,7 @@ def test_config_field_delete_non_admin_rejected( assert "role" in response.json().get("detail", {}).get("error", "").lower() -def test_config_field_delete_field_not_in_config( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_delete_field_not_in_config(client, auth_as, mock_prisma, monkeypatch): """If there is no general_settings row at all, returns 400 'not in config'.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -690,9 +636,7 @@ def test_config_callback_delete_happy_admin(client, auth_as, mock_prisma, monkey monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.post( - "/config/callback/delete", json={"callback_name": "langfuse"} - ) + response = client.post("/config/callback/delete", json={"callback_name": "langfuse"}) assert response.status_code == 200 # `deleted_at` is an ISO timestamp generated at request time — extend # the volatile set just for this assertion so dict-equality still works. @@ -705,9 +649,7 @@ def test_config_callback_delete_happy_admin(client, auth_as, mock_prisma, monkey } -def test_config_callback_delete_non_admin_rejected( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_callback_delete_non_admin_rejected(client, auth_as, mock_prisma, monkeypatch): """Non-admin caller is rejected with 400 not-allowed.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -717,9 +659,7 @@ def test_config_callback_delete_non_admin_rejected( monkeypatch.setattr(ps, "store_model_in_db", True) with auth_as(LitellmUserRoles.INTERNAL_USER): - response = client.post( - "/config/callback/delete", json={"callback_name": "langfuse"} - ) + response = client.post("/config/callback/delete", json={"callback_name": "langfuse"}) assert response.status_code == 400 assert "role" in response.json().get("detail", {}).get("error", "").lower() @@ -734,22 +674,15 @@ def test_config_callback_delete_not_found(client, auth_as, mock_prisma, monkeypa monkeypatch.setattr(ps, "store_model_in_db", True) fake_proxy_config = MagicMock() - fake_proxy_config.get_config = AsyncMock( - return_value={"litellm_settings": {"success_callback": ["slack"]}} - ) + fake_proxy_config.get_config = AsyncMock(return_value={"litellm_settings": {"success_callback": ["slack"]}}) monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.post( - "/config/callback/delete", json={"callback_name": "langfuse"} - ) + response = client.post("/config/callback/delete", json={"callback_name": "langfuse"}) # The handler re-raises HTTPException(404) verbatim (only generic # `Exception` becomes a 500 ProxyException), so pin 404 strictly. assert response.status_code == 404 - assert ( - "langfuse" in str(response.json()).lower() - or "not found" in str(response.json()).lower() - ) + assert "langfuse" in str(response.json()).lower() or "not found" in str(response.json()).lower() # --------------------------------------------------------------------------- @@ -813,10 +746,7 @@ def test_get_config_callbacks_internal_error(client, auth_as, mock_prisma, monke with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/get/config/callbacks") assert response.status_code >= 400 - assert ( - "boom" in str(response.json()).lower() - or "error" in str(response.json()).lower() - ) + assert "boom" in str(response.json()).lower() or "error" in str(response.json()).lower() _CALLBACK_ENV_FIXTURE = { @@ -850,14 +780,10 @@ def _install_callbacks_config(monkeypatch, mock_prisma): def _callback_variables(body: dict, name: str) -> dict: - return next( - cb["variables"] for cb in body["callbacks"] if cb["name"] == name - ) + return next(cb["variables"] for cb in body["callbacks"] if cb["name"] == name) -def test_get_config_callbacks_redacts_secret_env_vars_for_view_only_admin( - client, auth_as, mock_prisma, monkeypatch -): +def test_get_config_callbacks_redacts_secret_env_vars_for_view_only_admin(client, auth_as, mock_prisma, monkeypatch): from litellm.proxy._types import LitellmUserRoles _install_callbacks_config(monkeypatch, mock_prisma) @@ -889,9 +815,7 @@ def test_get_config_callbacks_redacts_secret_env_vars_for_view_only_admin( assert otel_vars["OTEL_ENDPOINT"] == _CALLBACK_ENV_FIXTURE["OTEL_ENDPOINT"] -def test_get_config_callbacks_full_admin_still_sees_secret_env_vars( - client, auth_as, mock_prisma, monkeypatch -): +def test_get_config_callbacks_full_admin_still_sees_secret_env_vars(client, auth_as, mock_prisma, monkeypatch): from litellm.proxy._types import LitellmUserRoles _install_callbacks_config(monkeypatch, mock_prisma) @@ -912,9 +836,7 @@ def test_get_config_callbacks_full_admin_still_sees_secret_env_vars( assert otel_vars["OTEL_HEADERS"] == _CALLBACK_ENV_FIXTURE["OTEL_HEADERS"] -def test_get_config_callbacks_redacts_slack_webhook_urls_for_view_only_admin( - client, auth_as, mock_prisma, monkeypatch -): +def test_get_config_callbacks_redacts_slack_webhook_urls_for_view_only_admin(client, auth_as, mock_prisma, monkeypatch): from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -1035,9 +957,7 @@ def test_get_config_callbacks_redacts_email_alerting_vars_for_view_only_admin( assert admin_email["SMTP_HOST"] == "smtp.resend.com" -def test_get_config_callbacks_appends_runtime_only_callbacks( - client, auth_as, mock_prisma, monkeypatch -): +def test_get_config_callbacks_appends_runtime_only_callbacks(client, auth_as, mock_prisma, monkeypatch): """Runtime-registered callbacks (not in config) are appended as read_only rows.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -1060,6 +980,17 @@ def test_get_config_callbacks_appends_runtime_only_callbacks( import litellm monkeypatch.setattr(litellm, "callbacks", ["otel"]) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_callbacks_by_type", + MagicMock( + return_value={ + "success": [], + "failure": [], + "success_and_failure": ["otel"], + } + ), + ) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/get/config/callbacks") @@ -1075,17 +1006,16 @@ def test_get_config_callbacks_appends_runtime_only_callbacks( # Configured callback should NOT be marked read_only langfuse_cb = next(cb for cb in callbacks if cb["name"] == "langfuse") - assert langfuse_cb.get("read_only") != True + assert not langfuse_cb.get("read_only") # Runtime-only callback should be marked read_only otel_cb = next(cb for cb in callbacks if cb["name"] == "otel") assert otel_cb["read_only"] is True assert otel_cb["type"] == "success_and_failure" + assert {callback["name"] for callback in callbacks} == {"langfuse", "otel"} -def test_get_config_callbacks_deduplicates_configured_and_runtime( - client, auth_as, mock_prisma, monkeypatch -): +def test_get_config_callbacks_deduplicates_configured_and_runtime(client, auth_as, mock_prisma, monkeypatch): """When same callback is in both config and runtime, show only once as configured.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -1107,23 +1037,96 @@ def test_get_config_callbacks_deduplicates_configured_and_runtime( # Mock runtime: same callback registered that is also in config import litellm - original = litellm.success_callback - try: - litellm.success_callback = ["langfuse"] - with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get("/get/config/callbacks") - assert response.status_code == 200 - body = response.json() + monkeypatch.setattr(litellm, "success_callback", ["langfuse"]) + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_callbacks_by_type", + MagicMock( + return_value={ + "success": ["langfuse"], + "failure": [], + "success_and_failure": [], + } + ), + ) - callbacks = body["callbacks"] - langfuse_rows = [cb for cb in callbacks if cb["name"] == "langfuse"] + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + body = response.json() - # Should appear exactly once, not duplicated - assert len(langfuse_rows) == 1 - # And it should NOT be marked read_only (it's in config) - assert langfuse_rows[0].get("read_only") != True - finally: - litellm.success_callback = original + callbacks = body["callbacks"] + langfuse_rows = [cb for cb in callbacks if cb["name"] == "langfuse"] + + # Should appear exactly once, not duplicated + assert len(langfuse_rows) == 1 + # And it should NOT be marked read_only (it's in config) + assert not langfuse_rows[0].get("read_only") + assert {callback["name"] for callback in callbacks} == {"langfuse"} + + +def test_get_config_callbacks_excludes_internal_runtime_callbacks(client, auth_as, mock_prisma, monkeypatch): + """Proxy infrastructure callbacks are excluded from callback inventory.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": []}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + + class _InventoryTestGuardrail(CustomGuardrail): + pass + + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_callbacks_by_type", + MagicMock( + return_value={ + "success": ["langsmith", "deployment_callback_on_success", "cache"], + "failure": ["deployment_callback_on_failure"], + "success_and_failure": ["_ProxyDBLogger", "SkillsInjectionHook", "_InventoryTestGuardrail"], + } + ), + ) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_custom_loggers_for_type", + MagicMock(return_value=[_InventoryTestGuardrail(guardrail_name="inventory-test-guardrail")]), + ) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + + assert response.status_code == 200 + assert response.json()["callbacks"] == [ + { + "name": "langsmith", + "variables": { + "LANGSMITH_API_KEY": None, + "LANGSMITH_PROJECT": None, + "LANGSMITH_DEFAULT_RUN_NAME": None, + }, + "type": "success", + "read_only": True, + }, + ] def test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_admin( @@ -1151,6 +1154,17 @@ def test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_adm import litellm monkeypatch.setattr(litellm, "callbacks", ["otel"]) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_callbacks_by_type", + MagicMock( + return_value={ + "success": [], + "failure": [], + "success_and_failure": ["otel"], + } + ), + ) # View-only admin with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): @@ -1172,9 +1186,7 @@ def test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_adm admin_response = client.get("/get/config/callbacks") assert admin_response.status_code == 200 admin_body = admin_response.json() - admin_otel = next( - (cb for cb in admin_body["callbacks"] if cb["name"] == "otel"), None - ) + admin_otel = next((cb for cb in admin_body["callbacks"] if cb["name"] == "otel"), None) assert admin_otel is not None assert admin_otel["variables"]["OTEL_HEADERS"] == _CALLBACK_ENV_FIXTURE["OTEL_HEADERS"] @@ -1192,9 +1204,7 @@ def test_config_yaml_returns_demo_payload(client, auth_as): response = client.request("GET", "/config/yaml", json={}) shape = { "status": response.status_code, - "media_type_yaml": response.headers.get("content-type", "").startswith( - "application/json" - ), + "media_type_yaml": response.headers.get("content-type", "").startswith("application/json"), "has_body": len(response.content) > 0, } assert shape == { diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx index 3ef7dee01b0..3b987a93478 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx @@ -97,6 +97,23 @@ describe("LoggingCallbacksTable", () => { expect(onDelete).toHaveBeenCalledWith(callback); }); + it("hides the actions menu for read-only runtime callback rows", () => { + render( + , + ); + expect(screen.getByTestId("callback-actions-langfuse-success")).toBeInTheDocument(); + expect(screen.queryByTestId("callback-actions-datadog-success")).not.toBeInTheDocument(); + }); + // Regression: `/get_callbacks` returns the same `name` twice when a // callback is registered for both success and failure (e.g. `generic_api` // → POST to spend-log on both 200 and 4xx/5xx). The UI used to ignore