From 35a5381b83c98ad2770b48c9f76c927eae8b0efb Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 9 Mar 2026 15:01:18 -0700 Subject: [PATCH] fix(langfuse): prevent langfuse_secret_key from leaking into traces on 429 errors Three leak paths existed when a virtual key had per-key Langfuse credentials: 1. user_api_key_metadata.logging - already scrubbed (pre-existing fix) 2. user_api_key_auth_metadata.logging - new: scrub in scrub_sensitive_keys_in_metadata 3. user_api_key_team_metadata.callback_settings - new: scrub in scrub_sensitive_keys_in_metadata 4. user_api_key_auth field - new: scrub the full UserAPIKeyAuth object in metadata 5. modelParameters (langfuse_secret_key/public_key/host/litellm_logging_obj) - new: pop these from optional_params in langfuse.py before logging to Langfuse generations Verified manually: made two calls with a virtual key (rpm_limit=1), second call triggered 429, Langfuse trace shows no secret key in modelParameters or metadata. --- litellm/integrations/langfuse/langfuse.py | 21 ++++-- litellm/litellm_core_utils/litellm_logging.py | 32 ++++++++ .../test_langfuse_unit_tests.py | 74 +++++++++++++++++++ 3 files changed, 120 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 7bf97665fd2..202013e448a 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -20,16 +20,16 @@ from packaging.version import Version import litellm from litellm._logging import verbose_logger from litellm.constants import MAX_LANGFUSE_INITIALIZED_CLIENTS -from litellm.litellm_core_utils.core_helpers import ( - safe_deep_copy, - reconstruct_model_name, - filter_exceptions_from_params, -) -from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.integrations.langfuse.langfuse_mock_client import ( create_mock_langfuse_client, should_use_langfuse_mock, ) +from litellm.litellm_core_utils.core_helpers import ( + filter_exceptions_from_params, + reconstruct_model_name, + safe_deep_copy, +) +from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.llms.custom_httpx.http_handler import _get_httpx_client from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.langfuse import * @@ -87,7 +87,7 @@ def _extract_cache_read_input_tokens(usage_obj) -> int: ): cache_read_input_tokens = cached_tokens - return cache_read_input_tokens + return cache_read_input_tokens # type: ignore[return-value] class LangFuseLogger: @@ -293,6 +293,13 @@ class LangFuseLogger: tools = optional_params.pop("tools", None) # Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers) optional_params.pop("secret_fields", None) + # Remove Langfuse credential keys - these are per-key/team callback vars that should + # not appear in modelParameters (they would expose secret keys in traces) + optional_params.pop("langfuse_secret_key", None) + optional_params.pop("langfuse_secret", None) + optional_params.pop("langfuse_public_key", None) + optional_params.pop("langfuse_host", None) + optional_params.pop("litellm_logging_obj", None) if functions is not None: prompt["functions"] = functions if tools is not None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 6f587abcdf1..2eb7693aead 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5516,6 +5516,38 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]): metadata["user_api_key_metadata"] = cleaned_user_api_key_metadata litellm_params["metadata"] = metadata + ## check user_api_key_auth_metadata for sensitive logging keys (same as user_api_key_metadata) + if "user_api_key_auth_metadata" in metadata and isinstance( + metadata["user_api_key_auth_metadata"], dict + ): + cleaned_auth_metadata = {} + for k, v in metadata["user_api_key_auth_metadata"].items(): + if k == "logging": # prevent logging callback credentials + cleaned_auth_metadata[k] = "scrubbed_by_litellm_for_sensitive_keys" + else: + cleaned_auth_metadata[k] = v + metadata["user_api_key_auth_metadata"] = cleaned_auth_metadata + litellm_params["metadata"] = metadata + + ## check user_api_key_team_metadata for sensitive callback_settings (contains callback_vars with secrets) + if "user_api_key_team_metadata" in metadata and isinstance( + metadata["user_api_key_team_metadata"], dict + ): + cleaned_team_metadata = {} + for k, v in metadata["user_api_key_team_metadata"].items(): + if k == "callback_settings": # contains callback_vars with credentials + cleaned_team_metadata[k] = "scrubbed_by_litellm_for_sensitive_keys" + else: + cleaned_team_metadata[k] = v + metadata["user_api_key_team_metadata"] = cleaned_team_metadata + litellm_params["metadata"] = metadata + + ## scrub user_api_key_auth - this is the full UserAPIKeyAuth object which contains + ## metadata.logging[].callback_vars with langfuse_secret_key and other credentials + if "user_api_key_auth" in metadata: + metadata["user_api_key_auth"] = "scrubbed_by_litellm_for_sensitive_keys" + litellm_params["metadata"] = metadata + return litellm_params diff --git a/tests/logging_callback_tests/test_langfuse_unit_tests.py b/tests/logging_callback_tests/test_langfuse_unit_tests.py index 21d18fefade..421c13d710c 100644 --- a/tests/logging_callback_tests/test_langfuse_unit_tests.py +++ b/tests/logging_callback_tests/test_langfuse_unit_tests.py @@ -512,3 +512,77 @@ def test_masking_function_not_in_metadata_when_not_provided(): # Original metadata should be unchanged assert result["metadata"]["some_key"] == "some_value" + + +def test_scrub_sensitive_keys_user_api_key_auth_metadata(): + """ + Test that scrub_sensitive_keys_in_metadata scrubs the 'logging' key from + user_api_key_auth_metadata to prevent langfuse_secret_key from leaking into + Langfuse traces on 429 errors. + """ + from litellm.litellm_core_utils.litellm_logging import scrub_sensitive_keys_in_metadata + + litellm_params = { + "metadata": { + "user_api_key_auth_metadata": { + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-lf-test", + "langfuse_secret_key": "sk-lf-secret", + }, + } + ], + "other_key": "other_value", + }, + "some_other_metadata": "value", + } + } + + result = scrub_sensitive_keys_in_metadata(litellm_params) + + auth_metadata = result["metadata"]["user_api_key_auth_metadata"] + # logging key should be scrubbed (contains callback credentials) + assert auth_metadata["logging"] == "scrubbed_by_litellm_for_sensitive_keys" + # other keys should remain + assert auth_metadata["other_key"] == "other_value" + # unrelated metadata should be unchanged + assert result["metadata"]["some_other_metadata"] == "value" + + +def test_scrub_sensitive_keys_user_api_key_team_metadata(): + """ + Test that scrub_sensitive_keys_in_metadata scrubs the 'callback_settings' key from + user_api_key_team_metadata to prevent langfuse_secret_key from leaking into + Langfuse traces (deprecated team callback format). + """ + from litellm.litellm_core_utils.litellm_logging import scrub_sensitive_keys_in_metadata + + litellm_params = { + "metadata": { + "user_api_key_team_metadata": { + "callback_settings": { + "callback_vars": { + "langfuse_public_key": "pk-lf-test", + "langfuse_secret_key": "sk-lf-secret", + }, + "success_callback": ["langfuse"], + "failure_callback": [], + }, + "team_name": "my-team", + }, + "some_other_metadata": "value", + } + } + + result = scrub_sensitive_keys_in_metadata(litellm_params) + + team_metadata = result["metadata"]["user_api_key_team_metadata"] + # callback_settings should be scrubbed (contains callback_vars with credentials) + assert team_metadata["callback_settings"] == "scrubbed_by_litellm_for_sensitive_keys" + # other keys should remain + assert team_metadata["team_name"] == "my-team" + # unrelated metadata should be unchanged + assert result["metadata"]["some_other_metadata"] == "value"