From 2d5ae35a8517aa480049d37600a531d0eb6d85b8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 6 Nov 2025 12:38:47 -0800 Subject: [PATCH 1/6] Show all callbacks on UI --- litellm/proxy/proxy_server.py | 123 ++++--- tests/proxy_unit_tests/test_proxy_server.py | 163 ++++++++ .../src/components/settings.test.tsx | 136 +++++++ .../src/components/settings.tsx | 348 ++++++++++-------- 4 files changed, 548 insertions(+), 222 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/settings.test.tsx diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bed3d218df4..4a0d39f4518 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9572,8 +9572,61 @@ async def get_config(): # noqa: PLR0915 _general_settings = config_data.get("general_settings", {}) environment_variables = config_data.get("environment_variables", {}) - # check if "langfuse" in litellm_settings + # Helper function to process callbacks and get environment variables + def process_callback(_callback: str, callback_type: str) -> dict: + """Process a single callback and return its data with environment variables""" + if _callback == "langfuse" or _callback == "langfuse_otel": + env_vars = [ + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LANGFUSE_HOST", + ] + elif _callback == "openmeter": + env_vars = [ + "OPENMETER_API_KEY", + ] + elif _callback == "braintrust": + env_vars = [ + "BRAINTRUST_API_KEY", + "BRAINTRUST_API_BASE", + ] + elif _callback == "traceloop": + env_vars = ["TRACELOOP_API_KEY"] + elif _callback == "custom_callback_api": + env_vars = ["GENERIC_LOGGER_ENDPOINT"] + elif _callback == "otel": + env_vars = ["OTEL_EXPORTER", "OTEL_ENDPOINT", "OTEL_HEADERS"] + elif _callback == "langsmith": + env_vars = [ + "LANGSMITH_API_KEY", + "LANGSMITH_PROJECT", + "LANGSMITH_DEFAULT_RUN_NAME", + ] + else: + env_vars = [] + + env_vars_dict = {} + for _var in env_vars: + env_variable = environment_variables.get(_var, None) + if env_variable is None: + env_vars_dict[_var] = None + else: + # decode + decrypt the value + decrypted_value = decrypt_value_helper( + value=env_variable, key=_var + ) + env_vars_dict[_var] = decrypted_value + + return { + "name": _callback, + "variables": env_vars_dict, + "type": callback_type + } + _success_callbacks = _litellm_settings.get("success_callback", []) + _failure_callbacks = _litellm_settings.get("failure_callback", []) + _generic_callbacks = _litellm_settings.get("callbacks", []) + _data_to_return = [] """ [ @@ -9584,70 +9637,20 @@ async def get_config(): # noqa: PLR0915 "LANGFUSE_SECRET_KEY": "value", "LANGFUSE_HOST": "value" }, + "type": "success" } ] """ + for _callback in _success_callbacks: - if _callback != "langfuse": - if _callback == "openmeter": - env_vars = [ - "OPENMETER_API_KEY", - ] - elif _callback == "braintrust": - env_vars = [ - "BRAINTRUST_API_KEY", - "BRAINTRUST_API_BASE", - ] - elif _callback == "traceloop": - env_vars = ["TRACELOOP_API_KEY"] - elif _callback == "custom_callback_api": - env_vars = ["GENERIC_LOGGER_ENDPOINT"] - elif _callback == "otel": - env_vars = ["OTEL_EXPORTER", "OTEL_ENDPOINT", "OTEL_HEADERS"] - elif _callback == "langsmith": - env_vars = [ - "LANGSMITH_API_KEY", - "LANGSMITH_PROJECT", - "LANGSMITH_DEFAULT_RUN_NAME", - ] - else: - env_vars = [] - - env_vars_dict = {} - for _var in env_vars: - env_variable = environment_variables.get(_var, None) - if env_variable is None: - env_vars_dict[_var] = None - else: - # decode + decrypt the value - decrypted_value = decrypt_value_helper( - value=env_variable, key=_var - ) - env_vars_dict[_var] = decrypted_value - - _data_to_return.append({"name": _callback, "variables": env_vars_dict}) - elif _callback == "langfuse": - _langfuse_vars = [ - "LANGFUSE_PUBLIC_KEY", - "LANGFUSE_SECRET_KEY", - "LANGFUSE_HOST", - ] - _langfuse_env_vars = {} - for _var in _langfuse_vars: - env_variable = environment_variables.get(_var, None) - if env_variable is None: - _langfuse_env_vars[_var] = None - else: - # decode + decrypt the value - decrypted_value = decrypt_value_helper( - value=env_variable, key=_var - ) - _langfuse_env_vars[_var] = decrypted_value - - _data_to_return.append( - {"name": _callback, "variables": _langfuse_env_vars} - ) + _data_to_return.append(process_callback(_callback, "success")) + + for _callback in _failure_callbacks: + _data_to_return.append(process_callback(_callback, "failure")) + + for _callback in _generic_callbacks: + _data_to_return.append(process_callback(_callback, "generic")) # Check if slack alerting is on _alerting = _general_settings.get("alerting", []) diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 1f4bf806c16..3aeb8a840c9 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2371,3 +2371,166 @@ def test_non_root_ui_path_logic(monkeypatch, tmp_path, ui_exists, ui_has_content error_calls = [call[0][0] for call in mock_logger.error.call_args_list] assert any("Path exists:" in call for call in error_calls) assert mock_logger.info.call_count == 0 + + +@pytest.mark.asyncio +async def test_get_config_callbacks_with_all_types(client_no_auth): + """ + Test that /get/config/callbacks returns all three callback types: + - success_callback with type="success" + - failure_callback with type="failure" + - callbacks (generic) with type="generic" + """ + from litellm.proxy.proxy_server import ProxyConfig + + # Create a mock config with all three callback types + mock_config_data = { + "litellm_settings": { + "success_callback": ["langfuse", "braintrust"], + "failure_callback": ["sentry"], + "callbacks": ["otel", "langsmith"] + }, + "environment_variables": { + "LANGFUSE_PUBLIC_KEY": "test-public-key", + "LANGFUSE_SECRET_KEY": "test-secret-key", + "LANGFUSE_HOST": "https://test.langfuse.com", + "BRAINTRUST_API_KEY": "test-braintrust-key", + "OTEL_EXPORTER": "otlp", + "OTEL_ENDPOINT": "http://localhost:4317", + "LANGSMITH_API_KEY": "test-langsmith-key", + }, + "general_settings": {} + } + + proxy_config = getattr(litellm.proxy.proxy_server, "proxy_config") + + with patch.object( + proxy_config, "get_config", new=AsyncMock(return_value=mock_config_data) + ), patch( + "litellm.proxy.proxy_server.decrypt_value_helper", + side_effect=lambda value, key=None: value + ): + response = client_no_auth.get("/get/config/callbacks") + + assert response.status_code == 200 + result = response.json() + + # Verify response structure + assert "status" in result + assert result["status"] == "success" + assert "callbacks" in result + + callbacks = result["callbacks"] + + # Verify we have all 5 callbacks (2 success + 1 failure + 2 generic) + assert len(callbacks) == 5 + + # Group callbacks by type + success_callbacks = [cb for cb in callbacks if cb.get("type") == "success"] + failure_callbacks = [cb for cb in callbacks if cb.get("type") == "failure"] + generic_callbacks = [cb for cb in callbacks if cb.get("type") == "generic"] + + # Verify all callbacks have required fields + for callback in callbacks: + assert "name" in callback + assert "variables" in callback + assert "type" in callback + assert callback["type"] in ["success", "failure", "generic"] + + # Verify success callbacks + assert len(success_callbacks) == 2 + success_names = [cb["name"] for cb in success_callbacks] + assert "langfuse" in success_names + assert "braintrust" in success_names + + # Verify failure callbacks + assert len(failure_callbacks) == 1 + assert failure_callbacks[0]["name"] == "sentry" + + # Verify generic callbacks + assert len(generic_callbacks) == 2 + generic_names = [cb["name"] for cb in generic_callbacks] + assert "otel" in generic_names + assert "langsmith" in generic_names + + +@pytest.mark.asyncio +async def test_get_config_callbacks_environment_variables(client_no_auth): + """ + Test that /get/config/callbacks correctly includes environment variables + for each callback type with proper decryption. + """ + from litellm.proxy.proxy_server import ProxyConfig + + # Create a mock config with callbacks and their env vars + mock_config_data = { + "litellm_settings": { + "success_callback": ["langfuse"], + "failure_callback": [], + "callbacks": ["otel"] + }, + "environment_variables": { + "LANGFUSE_PUBLIC_KEY": "encrypted-public-key", + "LANGFUSE_SECRET_KEY": "encrypted-secret-key", + "LANGFUSE_HOST": "https://cloud.langfuse.com", + "OTEL_EXPORTER": "otlp", + "OTEL_ENDPOINT": "http://localhost:4317", + "OTEL_HEADERS": "key=value", + }, + "general_settings": {} + } + + # Mock decrypt to prepend "decrypted-" to values + def mock_decrypt(value, key=None): + if value and isinstance(value, str) and "encrypted" in value: + return f"decrypted-{value}" + return value + + proxy_config = getattr(litellm.proxy.proxy_server, "proxy_config") + + with patch.object( + proxy_config, "get_config", new=AsyncMock(return_value=mock_config_data) + ), patch( + "litellm.proxy.proxy_server.decrypt_value_helper", + side_effect=mock_decrypt + ): + response = client_no_auth.get("/get/config/callbacks") + + assert response.status_code == 200 + result = response.json() + + callbacks = result["callbacks"] + + # Find langfuse callback (success type) + langfuse_callback = next( + (cb for cb in callbacks if cb["name"] == "langfuse"), None + ) + assert langfuse_callback is not None + assert langfuse_callback["type"] == "success" + assert "variables" in langfuse_callback + + # Verify langfuse env vars are present and decrypted + langfuse_vars = langfuse_callback["variables"] + assert "LANGFUSE_PUBLIC_KEY" in langfuse_vars + assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == "decrypted-encrypted-public-key" + assert "LANGFUSE_SECRET_KEY" in langfuse_vars + assert langfuse_vars["LANGFUSE_SECRET_KEY"] == "decrypted-encrypted-secret-key" + assert "LANGFUSE_HOST" in langfuse_vars + assert langfuse_vars["LANGFUSE_HOST"] == "https://cloud.langfuse.com" + + # Find otel callback (generic type) + otel_callback = next( + (cb for cb in callbacks if cb["name"] == "otel"), None + ) + assert otel_callback is not None + assert otel_callback["type"] == "generic" + assert "variables" in otel_callback + + # Verify otel env vars are present + otel_vars = otel_callback["variables"] + assert "OTEL_EXPORTER" in otel_vars + assert otel_vars["OTEL_EXPORTER"] == "otlp" + assert "OTEL_ENDPOINT" in otel_vars + assert otel_vars["OTEL_ENDPOINT"] == "http://localhost:4317" + assert "OTEL_HEADERS" in otel_vars + assert otel_vars["OTEL_HEADERS"] == "key=value" diff --git a/ui/litellm-dashboard/src/components/settings.test.tsx b/ui/litellm-dashboard/src/components/settings.test.tsx new file mode 100644 index 00000000000..72a265eadd9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/settings.test.tsx @@ -0,0 +1,136 @@ +import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import { describe, expect, it, beforeAll, beforeEach, vi } from "vitest"; +import Settings from "./settings"; +import * as networking from "./networking"; + +beforeAll(() => { + Object.defineProperty(window, "matchMedia", { + writable: true, + value: (query: string) => ({ + matches: false, + media: query, + onchange: null, + addListener: () => {}, + removeListener: () => {}, + addEventListener: () => {}, + removeEventListener: () => {}, + dispatchEvent: () => true, + }), + }); +}); + +const mockCallbacksData = { + callbacks: [ + { + name: "langfuse", + type: "success", + variables: { + LANGFUSE_PUBLIC_KEY: "test_key", + LANGFUSE_SECRET_KEY: "test_secret", + }, + }, + { + name: "datadog", + type: "success", + variables: { + DD_API_KEY: "test_dd_key", + }, + }, + ], + available_callbacks: [ + { + litellm_callback_name: "langfuse", + ui_callback_name: "Langfuse", + litellm_callback_params: ["LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY"], + }, + { + litellm_callback_name: "datadog", + ui_callback_name: "Datadog", + litellm_callback_params: ["DD_API_KEY"], + }, + ], + alerts: [], +}; + +describe("Settings", () => { + it("should render the settings page", () => { + render(); + }); +}); + +describe("Logging Callbacks Section", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should display the list of active callbacks", async () => { + vi.spyOn(networking, "getCallbacksCall").mockResolvedValue(mockCallbacksData); + + render(); + + await waitFor( + () => { + expect(screen.getByText("langfuse")).toBeInTheDocument(); + expect(screen.getByText("datadog")).toBeInTheDocument(); + }, + { timeout: 3000 }, + ); + }); + + it("should open add callback modal and display form", async () => { + vi.spyOn(networking, "getCallbacksCall").mockResolvedValue(mockCallbacksData); + + render(); + + await waitFor(() => { + expect(screen.getByText("langfuse")).toBeInTheDocument(); + }); + + const addButton = screen.getByText("Add Callback"); + fireEvent.click(addButton); + + await waitFor(() => { + expect(screen.getByText("Add Logging Callback")).toBeInTheDocument(); + expect(screen.getByText("LiteLLM Docs: Logging")).toBeInTheDocument(); + }); + }); + + it("should successfully delete a callback", async () => { + const getCallbacksSpy = vi.spyOn(networking, "getCallbacksCall").mockResolvedValue(mockCallbacksData); + const deleteCallbackSpy = vi.spyOn(networking, "deleteCallback").mockResolvedValue(undefined); + + const { container } = render( + , + ); + + await waitFor(() => { + expect(screen.getByText("langfuse")).toBeInTheDocument(); + }); + + const trashIcons = container.querySelectorAll("svg"); + const trashIcon = Array.from(trashIcons).find((svg) => { + const parentElement = svg.parentElement; + return parentElement?.className.includes("text-red") || parentElement?.outerHTML.includes("red"); + }); + + expect(trashIcon).toBeDefined(); + if (trashIcon && trashIcon.parentElement) { + fireEvent.click(trashIcon.parentElement); + } + + await waitFor(() => { + const modalText = screen.getByText((content, element) => { + return element?.tagName.toLowerCase() === "p" && content.includes("Are you sure you want to delete"); + }); + expect(modalText).toBeInTheDocument(); + }); + + const deleteButton = screen.getByRole("button", { name: "Delete" }); + fireEvent.click(deleteButton); + + await waitFor(() => { + expect(deleteCallbackSpy).toHaveBeenCalledWith("test-token", "langfuse"); + expect(getCallbacksSpy).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 727c8b9b511..a275214918c 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -19,11 +19,12 @@ import { Tab, SelectItem, Icon, + Badge, } from "@tremor/react"; import { PencilAltIcon, TrashIcon } from "@heroicons/react/outline"; -import { Modal, Typography, Form, Input, Select, Button as Button2 } from "antd"; +import { Modal, Typography, Form, Input, Select, Button as Button2, Tooltip } from "antd"; import NotificationsManager from "./molecules/notifications_manager"; import EmailSettings from "./email_settings"; @@ -32,10 +33,7 @@ const { Title, Paragraph } = Typography; import { getCallbacksCall, setCallbacksCall, serviceHealthCheck, deleteCallback } from "./networking"; import AlertingSettings from "./alerting/alerting_settings"; import FormItem from "antd/es/form/FormItem"; -import { - CALLBACK_CONFIGS, - getCallbackById, -} from "./callback_info_helpers"; +import { CALLBACK_CONFIGS, getCallbackById } from "./callback_info_helpers"; import { parseErrorMessage } from "./shared/errorUtils"; interface SettingsPageProps { accessToken: string | null; @@ -60,6 +58,7 @@ interface AlertingVariables { interface AlertingObject { name: string; + type?: "success" | "failure" | "generic"; variables: AlertingVariables; } @@ -216,10 +215,10 @@ const Settings: React.FC = ({ accessToken, userRole, userID, const handleSelectedCallbackChange = (callbackName: string) => { setSelectedCallback(callbackName); - + // Get the callback configuration using the new clean structure const callbackConfig = getCallbackById(callbackName); - + // Get the parameters from the callback configuration if (callbackConfig?.dynamic_params) { const params = Object.keys(callbackConfig.dynamic_params); @@ -228,7 +227,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID, setSelectedCallbackParams([]); } }; - + const handleSaveAlerts = async () => { if (!accessToken) { return; @@ -416,54 +415,94 @@ const Settings: React.FC = ({ accessToken, userRole, userID, Active Logging Callbacks - + Callback Name - {/* Callback Env Vars */} + Callback Type + Actions - {callbacks.map((callback, index) => ( - - - {callback.name} - - - - { - setSelectedEditCallback(callback); - setShowEditCallback(true); - }} - /> - handleDeleteCallback(callback.name)} - className="text-red-500 hover:text-red-700 cursor-pointer" - /> - - - - - ))} + {callbacks.map((callback, index) => { + const canEdit = !callback.type || callback.type === "success"; + const tooltipMessage = + callback.type === "failure" + ? "Modifications and deletion of failure type callbacks are not yet supported in the UI" + : callback.type === "generic" + ? "Modifications and deletion of generic type callbacks are not yet supported in the UI" + : ""; + + const getBadgeColor = (type?: string) => { + if (type === "success") return "green"; + if (type === "failure") return "red"; + if (type === "generic") return "blue"; + return "gray"; + }; + + return ( + + + {callback.name} + + + {callback.type ? ( + {callback.type} + ) : ( + success + )} + + +
+ + { + if (canEdit) { + setSelectedEditCallback(callback); + setShowEditCallback(true); + } + }} + className={canEdit ? "cursor-pointer" : "opacity-40 cursor-not-allowed"} + /> + + + { + if (canEdit) { + handleDeleteCallback(callback.name); + } + }} + className={ + canEdit + ? "text-red-500 hover:text-red-700 cursor-pointer" + : "text-red-300 opacity-40 cursor-not-allowed" + } + /> + + +
+ + + ); + })}
@@ -594,124 +633,109 @@ const Settings: React.FC = ({ accessToken, userRole, userID, wrapperCol={{ span: 16 }} labelAlign="left" > - + - (option?.children?.toString() ?? "") - .toLowerCase() - .includes(input.toLowerCase()) - } - onChange={(value) => { - handleSelectedCallbackChange(value); - }} - > - {CALLBACK_CONFIGS.map((callbackConfig) => ( - -
-
- {/* eslint-disable-next-line @next/next/no-img-element */} - {`${callbackConfig.displayName} { - e.currentTarget.style.display = 'none'; - }} - /> -
- - {callbackConfig.displayName} - + {CALLBACK_CONFIGS.map((callbackConfig) => ( + +
+
+ {/* eslint-disable-next-line @next/next/no-img-element */} + {`${callbackConfig.displayName} { + e.currentTarget.style.display = "none"; + }} + />
- - ))} - - + {callbackConfig.displayName} +
+ + ))} + + - {selectedCallbackParams && selectedCallbackParams.length > 0 && ( -
- {selectedCallbackParams.map((param) => { - // Get the callback configuration to look up parameter types - const callbackConfig = getCallbackById(selectedCallback || ''); - const paramType = callbackConfig?.dynamic_params[param] || "text"; - - const fieldLabel = param.replace(/_/g, " ").replace(/\b\w/g, l => l.toUpperCase()); - - return ( - - {fieldLabel} - * - - } - name={param} - key={param} - className="mb-4" - rules={[ - { - required: true, - message: `Please enter the ${fieldLabel.toLowerCase()}`, - }, - ]} - > - {paramType === "password" ? ( - - ) : paramType === "number" ? ( - - ) : ( - - )} - - ); - })} -
- )} + {selectedCallbackParams && selectedCallbackParams.length > 0 && ( +
+ {selectedCallbackParams.map((param) => { + // Get the callback configuration to look up parameter types + const callbackConfig = getCallbackById(selectedCallback || ""); + const paramType = callbackConfig?.dynamic_params[param] || "text"; -
- - - Add Callback - + const fieldLabel = param.replace(/_/g, " ").replace(/\b\w/g, (l) => l.toUpperCase()); + + return ( + + {fieldLabel} + * + + } + name={param} + key={param} + className="mb-4" + rules={[ + { + required: true, + message: `Please enter the ${fieldLabel.toLowerCase()}`, + }, + ]} + > + {paramType === "password" ? ( + + ) : paramType === "number" ? ( + + ) : ( + + )} + + ); + })}
+ )} + +
+ + Add Callback +
From 0af3a51a974130009027355baa51b16750741d8b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 6 Nov 2025 12:48:04 -0800 Subject: [PATCH 2/6] Fix tests --- ui/litellm-dashboard/src/components/settings.test.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ui/litellm-dashboard/src/components/settings.test.tsx b/ui/litellm-dashboard/src/components/settings.test.tsx index 72a265eadd9..20dadf9d2a5 100644 --- a/ui/litellm-dashboard/src/components/settings.test.tsx +++ b/ui/litellm-dashboard/src/components/settings.test.tsx @@ -54,6 +54,10 @@ const mockCallbacksData = { describe("Settings", () => { it("should render the settings page", () => { + vi.spyOn(networking, "alertingSettingsCall").mockResolvedValue([]); + vi.spyOn(networking, "getEmailEventSettings").mockResolvedValue({ settings: [] }); + vi.spyOn(networking, "getCallbacksCall").mockResolvedValue(mockCallbacksData); + render(); }); }); @@ -61,6 +65,8 @@ describe("Settings", () => { describe("Logging Callbacks Section", () => { beforeEach(() => { vi.clearAllMocks(); + vi.spyOn(networking, "alertingSettingsCall").mockResolvedValue([]); + vi.spyOn(networking, "getEmailEventSettings").mockResolvedValue({ settings: [] }); }); it("should display the list of active callbacks", async () => { From 3a96c700b4173aca63a90283ecfce57bfc636f9d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 7 Nov 2025 15:02:41 -0800 Subject: [PATCH 3/6] Adjusted based on comments --- litellm/integrations/custom_logger.py | 38 +++++++++++++++++++ litellm/proxy/_types.py | 8 ++++ litellm/proxy/proxy_server.py | 37 +++--------------- tests/local_testing/test_custom_logger.py | 18 +++++++++ .../src/components/settings.tsx | 19 +++++++--- 5 files changed, 82 insertions(+), 38 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index fd8ab2bad9d..2a08408f7de 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -81,6 +81,44 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac self.turn_off_message_logging = turn_off_message_logging pass + @staticmethod + def get_callback_env_vars(callback_name: Optional[str] = None) -> List[str]: + """ + Return the environment variables associated with a given callback + name as defined in the proxy callback registry. + + Args: + callback_name: The name of the callback to look up. + + Returns: + List[str]: A list of required environment variable names. + """ + if callback_name is None: + return [] + + normalized_name = callback_name.lower() + + alias_map = { + "langfuse_otel": "langfuse", + } + lookup_name = alias_map.get(normalized_name, normalized_name) + + try: + from litellm.proxy._types import AllCallbacks + except Exception: + return [] + + callbacks = AllCallbacks() + callback_info = getattr(callbacks, lookup_name, None) + if callback_info is None: + return [] + + params = getattr(callback_info, "litellm_callback_params", None) + if not params: + return [] + + return list(params) + def log_pre_api_call(self, model, messages, kwargs): pass diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d739727ccab..397bfc9c3b8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2503,6 +2503,14 @@ class AllCallbacks(LiteLLMPydanticObjectBase): ui_callback_name="Lago Billing", ) + traceloop: CallbackOnUI = CallbackOnUI( + litellm_callback_name="traceloop", + litellm_callback_params=[ + "TRACELoop_API_KEY", + ], + ui_callback_name="Traceloop", + ) + class SpendLogsMetadata(TypedDict): """ diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4a0d39f4518..9a582814a55 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -153,6 +153,7 @@ from litellm.constants import ( ) from litellm.exceptions import RejectedRequestError from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_litellm_metadata_from_kwargs, @@ -9575,35 +9576,7 @@ async def get_config(): # noqa: PLR0915 # Helper function to process callbacks and get environment variables def process_callback(_callback: str, callback_type: str) -> dict: """Process a single callback and return its data with environment variables""" - if _callback == "langfuse" or _callback == "langfuse_otel": - env_vars = [ - "LANGFUSE_PUBLIC_KEY", - "LANGFUSE_SECRET_KEY", - "LANGFUSE_HOST", - ] - elif _callback == "openmeter": - env_vars = [ - "OPENMETER_API_KEY", - ] - elif _callback == "braintrust": - env_vars = [ - "BRAINTRUST_API_KEY", - "BRAINTRUST_API_BASE", - ] - elif _callback == "traceloop": - env_vars = ["TRACELOOP_API_KEY"] - elif _callback == "custom_callback_api": - env_vars = ["GENERIC_LOGGER_ENDPOINT"] - elif _callback == "otel": - env_vars = ["OTEL_EXPORTER", "OTEL_ENDPOINT", "OTEL_HEADERS"] - elif _callback == "langsmith": - env_vars = [ - "LANGSMITH_API_KEY", - "LANGSMITH_PROJECT", - "LANGSMITH_DEFAULT_RUN_NAME", - ] - else: - env_vars = [] + env_vars = CustomLogger.get_callback_env_vars(_callback) env_vars_dict = {} for _var in env_vars: @@ -9625,7 +9598,7 @@ async def get_config(): # noqa: PLR0915 _success_callbacks = _litellm_settings.get("success_callback", []) _failure_callbacks = _litellm_settings.get("failure_callback", []) - _generic_callbacks = _litellm_settings.get("callbacks", []) + _success_and_failure_callbacks = _litellm_settings.get("callbacks", []) _data_to_return = [] """ @@ -9649,8 +9622,8 @@ async def get_config(): # noqa: PLR0915 for _callback in _failure_callbacks: _data_to_return.append(process_callback(_callback, "failure")) - for _callback in _generic_callbacks: - _data_to_return.append(process_callback(_callback, "generic")) + for _callback in _success_and_failure_callbacks: + _data_to_return.append(process_callback(_callback, "success_and_failure")) # Check if slack alerting is on _alerting = _general_settings.get("alerting", []) diff --git a/tests/local_testing/test_custom_logger.py b/tests/local_testing/test_custom_logger.py index 00e7c2d5aa1..f3dc6a0a7a4 100644 --- a/tests/local_testing/test_custom_logger.py +++ b/tests/local_testing/test_custom_logger.py @@ -103,6 +103,24 @@ class TmpFunction: ) +def test_get_callback_env_vars(): + env_vars = CustomLogger.get_callback_env_vars("langfuse") + assert env_vars == [ + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LANGFUSE_HOST", + ] + + alias_env_vars = CustomLogger.get_callback_env_vars("langfuse_otel") + assert alias_env_vars == env_vars + + missing_env_vars = CustomLogger.get_callback_env_vars("does_not_exist") + assert missing_env_vars == [] + + none_env_vars = CustomLogger.get_callback_env_vars(None) + assert none_env_vars == [] + + @pytest.mark.asyncio async def test_async_chat_openai_stream(): try: diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index a275214918c..e8ac54e99dd 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -58,7 +58,7 @@ interface AlertingVariables { interface AlertingObject { name: string; - type?: "success" | "failure" | "generic"; + type?: "success" | "failure" | "success_and_failure"; variables: AlertingVariables; } @@ -430,17 +430,24 @@ const Settings: React.FC = ({ accessToken, userRole, userID, const tooltipMessage = callback.type === "failure" ? "Modifications and deletion of failure type callbacks are not yet supported in the UI" - : callback.type === "generic" - ? "Modifications and deletion of generic type callbacks are not yet supported in the UI" + : callback.type === "success_and_failure" + ? "Modifications and deletion of success and failure type callbacks are not yet supported in the UI" : ""; const getBadgeColor = (type?: string) => { if (type === "success") return "green"; if (type === "failure") return "red"; - if (type === "generic") return "blue"; + if (type === "success_and_failure") return "blue"; return "gray"; }; + const getBadgeLabel = (type?: string) => { + if (type === "success") return "Success Only"; + if (type === "failure") return "Failure Only"; + if (type === "success_and_failure") return "Success & Failure"; + return "Unknown"; + }; + return ( @@ -448,9 +455,9 @@ const Settings: React.FC = ({ accessToken, userRole, userID, {callback.type ? ( - {callback.type} + {getBadgeLabel(callback.type)} ) : ( - success + Unknown )} From 9be008a54b22d41d4f6727c10afe9335a6c7827a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 8 Nov 2025 14:50:53 -0800 Subject: [PATCH 4/6] Fixed typo --- litellm/proxy/_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 397bfc9c3b8..17be7c18aab 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2506,7 +2506,7 @@ class AllCallbacks(LiteLLMPydanticObjectBase): traceloop: CallbackOnUI = CallbackOnUI( litellm_callback_name="traceloop", litellm_callback_params=[ - "TRACELoop_API_KEY", + "TRACELOOP_API_KEY", ], ui_callback_name="Traceloop", ) From 67bca6dde4aaacda4c2acadab0bb5f61aaaeaa10 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 8 Nov 2025 15:00:58 -0800 Subject: [PATCH 5/6] Fix linting --- litellm/proxy/proxy_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9a582814a55..4acd2639d8f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9578,7 +9578,7 @@ async def get_config(): # noqa: PLR0915 """Process a single callback and return its data with environment variables""" env_vars = CustomLogger.get_callback_env_vars(_callback) - env_vars_dict = {} + env_vars_dict: dict[str, str | None] = {} for _var in env_vars: env_variable = environment_variables.get(_var, None) if env_variable is None: From 7833b3fdb4ed3d52dbf37d8f8c9c8923b6fd6386 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 10 Nov 2025 17:28:13 -0800 Subject: [PATCH 6/6] Addressing comments --- litellm/proxy/common_utils/callback_utils.py | 26 ++++++++++ litellm/proxy/proxy_server.py | 30 ++---------- tests/proxy_unit_tests/test_proxy_server.py | 26 +++++----- .../proxy/common_utils/test_callback_utils.py | 47 +++++++++++++++++++ 4 files changed, 90 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index fb7ada8ab10..eb312612779 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -3,8 +3,12 @@ from typing import Any, Dict, List, Literal, Optional import litellm from litellm import get_secret from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import CommonProxyErrors, LiteLLMPromptInjectionParams from litellm.proxy.types_utils.utils import get_instance_fn +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + decrypt_value_helper, +) blue_color_code = "\033[94m" reset_color_code = "\033[0m" @@ -382,3 +386,25 @@ def get_metadata_variable_name_from_kwargs( - LiteLLM is now moving to using `litellm_metadata` for our metadata """ return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" + +def process_callback(_callback: str, callback_type: str, environment_variables: dict) -> dict: + """Process a single callback and return its data with environment variables""" + env_vars = CustomLogger.get_callback_env_vars(_callback) + + env_vars_dict: dict[str, str | None] = {} + for _var in env_vars: + env_variable = environment_variables.get(_var, None) + if env_variable is None: + env_vars_dict[_var] = None + else: + # decode + decrypt the value + decrypted_value = decrypt_value_helper( + value=env_variable, key=_var + ) + env_vars_dict[_var] = decrypted_value + + return { + "name": _callback, + "variables": env_vars_dict, + "type": callback_type + } \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 090757da4ad..205609a645c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -47,6 +47,7 @@ from litellm.types.utils import ( TokenCountResponse, ) from litellm.utils import load_credentials_from_list +from litellm.proxy.common_utils.callback_utils import process_callback if TYPE_CHECKING: from aiohttp import ClientSession @@ -9582,29 +9583,6 @@ async def get_config(): # noqa: PLR0915 _general_settings = config_data.get("general_settings", {}) environment_variables = config_data.get("environment_variables", {}) - # Helper function to process callbacks and get environment variables - def process_callback(_callback: str, callback_type: str) -> dict: - """Process a single callback and return its data with environment variables""" - env_vars = CustomLogger.get_callback_env_vars(_callback) - - env_vars_dict: dict[str, str | None] = {} - for _var in env_vars: - env_variable = environment_variables.get(_var, None) - if env_variable is None: - env_vars_dict[_var] = None - else: - # decode + decrypt the value - decrypted_value = decrypt_value_helper( - value=env_variable, key=_var - ) - env_vars_dict[_var] = decrypted_value - - return { - "name": _callback, - "variables": env_vars_dict, - "type": callback_type - } - _success_callbacks = _litellm_settings.get("success_callback", []) _failure_callbacks = _litellm_settings.get("failure_callback", []) _success_and_failure_callbacks = _litellm_settings.get("callbacks", []) @@ -9626,13 +9604,13 @@ async def get_config(): # noqa: PLR0915 """ for _callback in _success_callbacks: - _data_to_return.append(process_callback(_callback, "success")) + _data_to_return.append(process_callback(_callback, "success", environment_variables)) for _callback in _failure_callbacks: - _data_to_return.append(process_callback(_callback, "failure")) + _data_to_return.append(process_callback(_callback, "failure", environment_variables)) for _callback in _success_and_failure_callbacks: - _data_to_return.append(process_callback(_callback, "success_and_failure")) + _data_to_return.append(process_callback(_callback, "success_and_failure", environment_variables)) # Check if slack alerting is on _alerting = _general_settings.get("alerting", []) diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 71b4503ff3c..17a6f8eb9ae 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2409,7 +2409,7 @@ async def test_get_config_callbacks_with_all_types(client_no_auth): Test that /get/config/callbacks returns all three callback types: - success_callback with type="success" - failure_callback with type="failure" - - callbacks (generic) with type="generic" + - callbacks (success_and_failure) with type="success_and_failure" """ from litellm.proxy.proxy_server import ProxyConfig @@ -2437,7 +2437,7 @@ async def test_get_config_callbacks_with_all_types(client_no_auth): with patch.object( proxy_config, "get_config", new=AsyncMock(return_value=mock_config_data) ), patch( - "litellm.proxy.proxy_server.decrypt_value_helper", + "litellm.proxy.common_utils.callback_utils.decrypt_value_helper", side_effect=lambda value, key=None: value ): response = client_no_auth.get("/get/config/callbacks") @@ -2452,20 +2452,20 @@ async def test_get_config_callbacks_with_all_types(client_no_auth): callbacks = result["callbacks"] - # Verify we have all 5 callbacks (2 success + 1 failure + 2 generic) + # Verify we have all 5 callbacks (2 success + 1 failure + 2 success_and_failure) assert len(callbacks) == 5 # Group callbacks by type success_callbacks = [cb for cb in callbacks if cb.get("type") == "success"] failure_callbacks = [cb for cb in callbacks if cb.get("type") == "failure"] - generic_callbacks = [cb for cb in callbacks if cb.get("type") == "generic"] + success_and_failure_callbacks = [cb for cb in callbacks if cb.get("type") == "success_and_failure"] # Verify all callbacks have required fields for callback in callbacks: assert "name" in callback assert "variables" in callback assert "type" in callback - assert callback["type"] in ["success", "failure", "generic"] + assert callback["type"] in ["success", "failure", "success_and_failure"] # Verify success callbacks assert len(success_callbacks) == 2 @@ -2477,11 +2477,11 @@ async def test_get_config_callbacks_with_all_types(client_no_auth): assert len(failure_callbacks) == 1 assert failure_callbacks[0]["name"] == "sentry" - # Verify generic callbacks - assert len(generic_callbacks) == 2 - generic_names = [cb["name"] for cb in generic_callbacks] - assert "otel" in generic_names - assert "langsmith" in generic_names + # Verify success_and_failure callbacks + assert len(success_and_failure_callbacks) == 2 + success_and_failure_names = [cb["name"] for cb in success_and_failure_callbacks] + assert "otel" in success_and_failure_names + assert "langsmith" in success_and_failure_names @pytest.mark.asyncio @@ -2521,7 +2521,7 @@ async def test_get_config_callbacks_environment_variables(client_no_auth): with patch.object( proxy_config, "get_config", new=AsyncMock(return_value=mock_config_data) ), patch( - "litellm.proxy.proxy_server.decrypt_value_helper", + "litellm.proxy.common_utils.callback_utils.decrypt_value_helper", side_effect=mock_decrypt ): response = client_no_auth.get("/get/config/callbacks") @@ -2548,12 +2548,12 @@ async def test_get_config_callbacks_environment_variables(client_no_auth): assert "LANGFUSE_HOST" in langfuse_vars assert langfuse_vars["LANGFUSE_HOST"] == "https://cloud.langfuse.com" - # Find otel callback (generic type) + # Find otel callback (success_and_failure type) otel_callback = next( (cb for cb in callbacks if cb["name"] == "otel"), None ) assert otel_callback is not None - assert otel_callback["type"] == "generic" + assert otel_callback["type"] == "success_and_failure" assert "variables" in otel_callback # Verify otel env vars are present diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index b9ed4b9b508..877f0092182 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -9,6 +9,9 @@ from litellm.proxy.common_utils.callback_utils import ( get_remaining_tokens_and_requests_from_request_data, ) +from unittest.mock import patch +from litellm.proxy.common_utils.callback_utils import process_callback + def test_get_remaining_tokens_and_requests_from_request_data(): model_group = "openrouter/google/gemini-2.0-flash-001" @@ -27,3 +30,47 @@ def test_get_remaining_tokens_and_requests_from_request_data(): f"x-litellm-key-remaining-requests-{expected_name}": 100, f"x-litellm-key-remaining-tokens-{expected_name}": 200, } + + +@patch( + "litellm.proxy.common_utils.callback_utils.CustomLogger.get_callback_env_vars", + return_value=["API_KEY", "MISSING_VAR"], +) +@patch( + "litellm.proxy.common_utils.callback_utils.decrypt_value_helper", + side_effect=lambda value, key: f"decrypted-{key}", +) +def test_process_callback_with_env_vars(mock_decrypt, mock_get_env_vars): + environment_variables = { + "API_KEY": "ENC_VALUE", + "UNUSED": "SHOULD_BE_IGNORED", + } + + result = process_callback( + _callback="my_callback", + callback_type="input", + environment_variables=environment_variables, + ) + + assert result["name"] == "my_callback" + assert result["type"] == "input" + assert result["variables"] == { + "API_KEY": "decrypted-API_KEY", + "MISSING_VAR": None, + } + + +@patch( + "litellm.proxy.common_utils.callback_utils.CustomLogger.get_callback_env_vars", + return_value=[], +) +def test_process_callback_with_no_required_env_vars(mock_get_env_vars): + result = process_callback( + _callback="another_callback", + callback_type="output", + environment_variables={"SHOULD_NOT_BE_USED": "VALUE"}, + ) + + assert result["name"] == "another_callback" + assert result["type"] == "output" + assert result["variables"] == {}