diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index daca48120cd..1f3a6961f39 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -131,8 +131,26 @@ class SensitiveDataMasker: return masked_data + def mask(self, data: object) -> object: + if isinstance(data, Mapping): + return self.mask_dict(dict(data)) + if isinstance(data, list): + return self._mask_sequence( + data, + 0, + DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER, + None, + False, + ) + return data + _default_masker = SensitiveDataMasker() +_error_masker = SensitiveDataMasker(visible_prefix=4, visible_suffix=0) + + +def mask_sensitive_structure(data: object) -> object: + return _error_masker.mask(data) def mask_sensitive_keys(data: Dict[str, Any], sensitive_fields: Set[str]) -> Dict[str, Any]: diff --git a/litellm/router.py b/litellm/router.py index 2d66bc3158d..593644f7f3d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -72,7 +72,11 @@ from litellm.litellm_core_utils.coroutine_checker import coroutine_checker from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging -from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker +from litellm.litellm_core_utils.secret_redaction import redact_string +from litellm.litellm_core_utils.sensitive_data_masker import ( + SensitiveDataMasker, + mask_sensitive_structure, +) from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_strategy.least_busy import LeastBusyLoggingHandler @@ -6085,7 +6089,9 @@ class Router: else: error_message = "model={}. context_window_fallbacks={}. fallbacks={}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format( - model_group, context_window_fallbacks, fallbacks + model_group, + mask_sensitive_structure(context_window_fallbacks), + mask_sensitive_structure(fallbacks), ) verbose_router_logger.info( msg="Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \ @@ -6119,7 +6125,9 @@ class Router: return response else: error_message = "model={}. content_policy_fallback={}. fallbacks={}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format( - model_group, content_policy_fallbacks, fallbacks + model_group, + mask_sensitive_structure(content_policy_fallbacks), + mask_sensitive_structure(fallbacks), ) verbose_router_logger.info( msg="Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \ @@ -6129,7 +6137,7 @@ class Router: if litellm.expose_router_debug_in_errors: e.message += "\n{}".format(error_message) if fallbacks is not None and model_group is not None: - verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}") + verbose_router_logger.debug(f"inside model fallbacks: {mask_sensitive_structure(fallbacks)}") ( fallback_model_group, generic_fallback_idx, @@ -6142,11 +6150,12 @@ class Router: fallback_model_group = fallbacks[generic_fallback_idx]["*"] if fallback_model_group is None: + masked_fallbacks = mask_sensitive_structure(fallbacks) verbose_router_logger.info( - f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}" + f"No fallback model group found for original model_group={model_group}. Fallbacks={masked_fallbacks}" ) if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors: - original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}" # type: ignore + original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={masked_fallbacks}" # type: ignore raise original_exception input_kwargs.update( @@ -6164,23 +6173,23 @@ class Router: return response except Exception as new_exception: parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) + fallback_failure_exception_str = redact_string(str(new_exception)) verbose_router_logger.error( "litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}\n{}\n\nDebug Information:\nCooldown Deployments={}".format( - str(new_exception), - traceback.format_exc(), + fallback_failure_exception_str, + redact_string(traceback.format_exc()), await _async_get_cooldown_deployments_with_debug_info( litellm_router_instance=self, parent_otel_span=parent_otel_span, ), ) ) - fallback_failure_exception_str = str(new_exception) if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors: # add the available fallbacks to the exception original_exception.message += ". Received Model Group={}\nAvailable Model Group Fallbacks={}".format( # type: ignore model_group, - fallback_model_group, + mask_sensitive_structure(fallback_model_group), ) if len(fallback_failure_exception_str) > 0: original_exception.message += ( # type: ignore diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 17d29ec0da8..eb92e9b1c29 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import litellm from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure from litellm.router_utils.add_retry_fallback_headers import ( add_fallback_headers_to_response, get_fallback_error_info, @@ -126,7 +127,7 @@ async def run_async_fallback( try: # LOGGING kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception) - verbose_router_logger.info(f"Falling back to model_group = {mg}") + verbose_router_logger.info(f"Falling back to model_group = {mask_sensitive_structure(mg)}") if isinstance(mg, str): kwargs["model"] = mg elif isinstance(mg, dict): diff --git a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py index 7239636fd48..9b0116e6979 100644 --- a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py @@ -197,3 +197,46 @@ def test_cost_per_token_fields_not_masked(): # Actual secrets must still be masked assert "*" in masked["api_key"] assert "*" in masked["access_token"] + + +def test_mask_sensitive_structure_passes_through_plain_topology_names(): + """Fallback groups are usually lists of model-group name strings; those + carry no secrets and must survive verbatim so opt-in debug output stays useful.""" + from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure + + assert mask_sensitive_structure(["gpt-4", "claude-3-haiku"]) == ["gpt-4", "claude-3-haiku"] + assert mask_sensitive_structure([{"gpt-3.5-turbo": ["claude-3-haiku"]}]) == [ + {"gpt-3.5-turbo": ["claude-3-haiku"]} + ] + assert mask_sensitive_structure(None) is None + + +def test_mask_sensitive_structure_masks_credentials_in_inline_fallback_dicts(): + """An inline-dict fallback can carry provider credentials; those values must be + masked before the structure is embedded in a client-facing error message.""" + from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure + + secret = "sk-INLINEFALLBACKSECRET1234567890" + aws_secret = "wJalrXUtnFEMIK7MDENGbPxRfiCYSECRETKEY" + masked = mask_sensitive_structure( + [{"model": "openai/gpt-4", "api_key": secret, "aws_secret_access_key": aws_secret}] + ) + + rendered = str(masked) + assert secret not in rendered + assert aws_secret not in rendered + # Non-secret keys stay visible so the fallback wiring remains debuggable + assert masked[0]["model"] == "openai/gpt-4" + assert "*" in masked[0]["api_key"] + assert "*" in masked[0]["aws_secret_access_key"] + + +def test_mask_sensitive_structure_masks_credentials_nested_in_config_shape(): + """Credentials nested inside the {group: [fallbacks]} config shape must also be masked.""" + from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure + + secret = "sk-NESTEDINLINESECRET0987654321" + masked = mask_sensitive_structure( + [{"primary-group": [{"model": "gpt-4o", "api_key": secret}]}] + ) + assert secret not in str(masked) diff --git a/tests/test_litellm/test_router_exception_redaction.py b/tests/test_litellm/test_router_exception_redaction.py index e40bf661da4..2066352e2ce 100644 --- a/tests/test_litellm/test_router_exception_redaction.py +++ b/tests/test_litellm/test_router_exception_redaction.py @@ -5,15 +5,21 @@ The Router historically appended internal config names (model_group, fallback_model_group, fallback failure detail, deployment timeouts, context_window_fallbacks dict, etc.) onto the message of the exception it re-raises. That message is then surfaced to clients by -ProxyException, leaking the proxy's internal wiring. +ProxyException, leaking the proxy's internal wiring and, when fallbacks +are configured as inline deployment dicts, the provider credentials +inside those dicts. The flag defaults to True to preserve historical behavior (no breaking change for existing deployments). Set it to False to redact -those strings from the raised exception's message. +those strings from the raised exception's message. Regardless of the +flag, provider credentials inside inline-dict fallbacks are now masked +so a raw api_key / aws_* value never reaches the client. These tests verify that with the flag ON (default) the historical -leak strings appear in the raised exception's message, and with the -flag OFF the proxy's internal wiring is redacted. +topology strings appear in the raised exception's message, with the +flag OFF the proxy's internal wiring is redacted, and that a raw +provider credential never appears in the message regardless of the +flag. Five leak sites are gated in `litellm/router.py`: @@ -40,6 +46,7 @@ _RECEIVED_MODEL_GROUP_PHRASE = "Received Model Group=" _AVAILABLE_FALLBACKS_PHRASE = "Available Model Group Fallbacks=" _CONTEXT_WINDOW_HINT_PHRASE = "context_window_fallbacks=" _INTERNAL_MODEL_GROUP_NAME = "all-anthropic/claude-secret-internal" +_FALLBACK_CREDENTIAL = "sk-INLINEFALLBACKSECRET1234567890" def _router_with_rate_limit_failure() -> Router: @@ -76,6 +83,37 @@ def _router_with_context_window_failure() -> Router: ) +def _router_with_credentialed_fallback() -> Router: + """Primary fails, and its fallback is an inline dict that carries a provider + api_key. When the fallback also fails, the router embeds that dict in the + exception message, which is where a raw credential would otherwise leak.""" + return Router( + model_list=[ + { + "model_name": _INTERNAL_MODEL_GROUP_NAME, + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "litellm.RateLimitError", + }, + "model_info": {"id": "secret-deployment-id"}, + }, + ], + fallbacks=[ + { + _INTERNAL_MODEL_GROUP_NAME: [ + { + "model": "gpt-4o", + "api_key": _FALLBACK_CREDENTIAL, + "mock_response": "litellm.RateLimitError", + } + ] + } + ], + num_retries=0, + ) + + @pytest.fixture(autouse=True) def _reset_expose_flag(): """Each test starts with the flag in its default (on) state.""" @@ -110,7 +148,8 @@ async def test_flag_off_does_not_leak_received_model_group(): @pytest.mark.asyncio -async def test_default_leaks_received_model_group(): +async def test_flag_on_shows_received_model_group(): + litellm.expose_router_debug_in_errors = True router = _router_with_rate_limit_failure() with pytest.raises(litellm.RateLimitError) as excinfo: await router.acompletion( @@ -142,7 +181,8 @@ async def test_flag_off_does_not_leak_context_window_fallback_hint(): @pytest.mark.asyncio -async def test_default_leaks_context_window_fallback_hint(): +async def test_flag_on_shows_context_window_fallback_hint(): + litellm.expose_router_debug_in_errors = True router = _router_with_context_window_failure() with pytest.raises(litellm.ContextWindowExceededError) as excinfo: await router.acompletion( @@ -153,7 +193,7 @@ async def test_default_leaks_context_window_fallback_hint(): assert _CONTEXT_WINDOW_HINT_PHRASE in msg, msg # Site 5 also fires for ContextWindow errors that exit the # orchestrator without fallback resolution, so the model_group - # name leaks under the default behavior. + # name is shown under the opt-in behavior. assert _INTERNAL_MODEL_GROUP_NAME in msg, msg @@ -192,7 +232,8 @@ async def test_flag_off_does_not_leak_when_no_fallback_group_found(): @pytest.mark.asyncio -async def test_default_leaks_when_no_fallback_group_found(): +async def test_flag_on_shows_when_no_fallback_group_found(): + litellm.expose_router_debug_in_errors = True router = Router( model_list=[ { @@ -258,7 +299,8 @@ async def test_flag_off_does_not_leak_deployment_timeout_debug(): @pytest.mark.asyncio -async def test_default_leaks_deployment_timeout_debug(): +async def test_flag_on_shows_deployment_timeout_debug(): + litellm.expose_router_debug_in_errors = True router = _router_with_plain_deployment() with pytest.raises(litellm.Timeout) as excinfo: await router.acompletion( @@ -298,7 +340,8 @@ async def test_flag_off_does_not_leak_content_policy_fallback_hint(): @pytest.mark.asyncio -async def test_default_leaks_content_policy_fallback_hint(): +async def test_flag_on_shows_content_policy_fallback_hint(): + litellm.expose_router_debug_in_errors = True router = _router_with_plain_deployment() with pytest.raises(litellm.ContentPolicyViolationError) as excinfo: await router.acompletion( @@ -309,3 +352,82 @@ async def test_default_leaks_content_policy_fallback_hint(): msg = excinfo.value.message assert "content_policy_fallback=" in msg, msg assert _INTERNAL_MODEL_GROUP_NAME in msg, msg + + +# --- Credential masking: raw provider keys never leak, either flag state ---- + + +@pytest.mark.asyncio +async def test_flag_off_hides_fallback_credentials(): + litellm.expose_router_debug_in_errors = False + router = _router_with_credentialed_fallback() + with pytest.raises(litellm.RateLimitError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + ) + msg = excinfo.value.message + assert _FALLBACK_CREDENTIAL not in msg, msg + assert _AVAILABLE_FALLBACKS_PHRASE not in msg, msg + + +@pytest.mark.asyncio +async def test_flag_on_masks_fallback_credentials(): + litellm.expose_router_debug_in_errors = True + router = _router_with_credentialed_fallback() + with pytest.raises(litellm.RateLimitError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + ) + msg = excinfo.value.message + # The raw credential must never appear, even though debug exposure is on + assert _FALLBACK_CREDENTIAL not in msg, msg + # The fallback wiring is still shown (masking preserves structure, it does + # not drop the whole message), so the api_key key name survives + assert "api_key" in msg, msg + + +@pytest.mark.asyncio +async def test_flag_on_scrubs_credential_from_inner_fallback_exception_string(): + """If the fallback attempt itself raises an exception whose message embeds a + raw provider credential (e.g. a provider SDK echoing back the api_key it was + called with), that string is re-embedded via `Error doing the fallback: ...` + on the terminal raise. The router must scrub known secret patterns from it. + The primary fails with a benign rate-limit; the fallback deployment fails + with an exception whose text contains the secret.""" + litellm.expose_router_debug_in_errors = True + inner_secret = "sk-INNERFALLBACKEXCEPTIONSECRET1234" + router = Router( + model_list=[ + { + "model_name": _INTERNAL_MODEL_GROUP_NAME, + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "litellm.RateLimitError", + }, + "model_info": {"id": "secret-deployment-id"}, + }, + { + "model_name": "fallback-group", + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": f"Exception: content_filter_policy - api_key={inner_secret}", + }, + "model_info": {"id": "fallback-deployment-id"}, + }, + ], + fallbacks=[{_INTERNAL_MODEL_GROUP_NAME: ["fallback-group"]}], + num_retries=0, + ) + with pytest.raises(litellm.RateLimitError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + ) + msg = excinfo.value.message + assert "Error doing the fallback:" in msg, msg + assert inner_secret not in msg, msg + assert "REDACTED" in msg, msg