fix(router): mask provider credentials embedded in fallback error messages (#32083)

The Router's async fallback orchestrator appended fallback structures
(fallback_model_group, fallbacks, context_window_fallbacks,
content_policy_fallbacks) and the inner fallback exception onto
original_exception.message before re-raising. That message is forwarded
verbatim by the proxy as ProxyException.message. When fallbacks are
configured as inline deployment dicts, the raw provider api_key /
aws_secret_access_key inside those dicts reached any authenticated
caller in the response body.

Route the fallback structures through a new mask_sensitive_structure
helper (reuses the existing SensitiveDataMasker), and wrap the inner
fallback exception string in the existing redact_string. Topology names
still render for debugging under the existing expose_router_debug_in_errors
opt-in; only credential values inside inline-dict fallbacks are masked.
The router's own verbose_router_logger calls that embedded the same
structures are updated alongside, so log output stays consistent with the
exception message.

Verified end-to-end against a real proxy hitting OpenAI: before, the
client response body contained the raw fallback api_key; after, with the
flag on, the api_key value is masked to a 4-char prefix while topology
names are still visible for the operator

litellm/litellm_core_utils/sensitive_data_masker.py

@@ -131,8 +131,26 @@ class SensitiveDataMasker:
 
         return masked_data
 
+    def mask(self, data: object) -> object:
+        if isinstance(data, Mapping):
+            return self.mask_dict(dict(data))
+        if isinstance(data, list):
+            return self._mask_sequence(
+                data,
+                0,
+                DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER,
+                None,
+                False,
+            )
+        return data
+
 
 _default_masker = SensitiveDataMasker()
+_error_masker = SensitiveDataMasker(visible_prefix=4, visible_suffix=0)
+
+
+def mask_sensitive_structure(data: object) -> object:
+    return _error_masker.mask(data)
 
 
 def mask_sensitive_keys(data: Dict[str, Any], sensitive_fields: Set[str]) -> Dict[str, Any]:

litellm/router.py

@@ -72,7 +72,11 @@ from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
-from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
+from litellm.litellm_core_utils.secret_redaction import redact_string
+from litellm.litellm_core_utils.sensitive_data_masker import (
+    SensitiveDataMasker,
+    mask_sensitive_structure,
+)
 from litellm.llms.openai_like.json_loader import JSONProviderRegistry
 from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
 from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
@@ -6085,7 +6089,9 @@ class Router:
 
                 else:
                     error_message = "model={}. context_window_fallbacks={}. fallbacks={}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
-                        model_group, context_window_fallbacks, fallbacks
+                        model_group,
+                        mask_sensitive_structure(context_window_fallbacks),
+                        mask_sensitive_structure(fallbacks),
                     )
                     verbose_router_logger.info(
                         msg="Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \
@@ -6119,7 +6125,9 @@ class Router:
                     return response
                 else:
                     error_message = "model={}. content_policy_fallback={}. fallbacks={}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
-                        model_group, content_policy_fallbacks, fallbacks
+                        model_group,
+                        mask_sensitive_structure(content_policy_fallbacks),
+                        mask_sensitive_structure(fallbacks),
                     )
                     verbose_router_logger.info(
                         msg="Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \
@@ -6129,7 +6137,7 @@ class Router:
                     if litellm.expose_router_debug_in_errors:
                         e.message += "\n{}".format(error_message)
             if fallbacks is not None and model_group is not None:
-                verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}")
+                verbose_router_logger.debug(f"inside model fallbacks: {mask_sensitive_structure(fallbacks)}")
                 (
                     fallback_model_group,
                     generic_fallback_idx,
@@ -6142,11 +6150,12 @@ class Router:
                     fallback_model_group = fallbacks[generic_fallback_idx]["*"]
 
                 if fallback_model_group is None:
+                    masked_fallbacks = mask_sensitive_structure(fallbacks)
                     verbose_router_logger.info(
-                        f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}"
+                        f"No fallback model group found for original model_group={model_group}. Fallbacks={masked_fallbacks}"
                     )
                     if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors:
-                        original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}"  # type: ignore
+                        original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={masked_fallbacks}"  # type: ignore
                     raise original_exception
 
                 input_kwargs.update(
@@ -6164,23 +6173,23 @@ class Router:
                 return response
         except Exception as new_exception:
             parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs)
+            fallback_failure_exception_str = redact_string(str(new_exception))
             verbose_router_logger.error(
                 "litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}\n{}\n\nDebug Information:\nCooldown Deployments={}".format(
-                    str(new_exception),
-                    traceback.format_exc(),
+                    fallback_failure_exception_str,
+                    redact_string(traceback.format_exc()),
                     await _async_get_cooldown_deployments_with_debug_info(
                         litellm_router_instance=self,
                         parent_otel_span=parent_otel_span,
                     ),
                 )
             )
-            fallback_failure_exception_str = str(new_exception)
 
         if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors:
             # add the available fallbacks to the exception
             original_exception.message += ". Received Model Group={}\nAvailable Model Group Fallbacks={}".format(  # type: ignore
                 model_group,
-                fallback_model_group,
+                mask_sensitive_structure(fallback_model_group),
             )
             if len(fallback_failure_exception_str) > 0:
                 original_exception.message += (  # type: ignore

litellm/router_utils/fallback_event_handlers.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 import litellm
 from litellm._logging import verbose_router_logger
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
 from litellm.router_utils.add_retry_fallback_headers import (
     add_fallback_headers_to_response,
     get_fallback_error_info,
@@ -126,7 +127,7 @@ async def run_async_fallback(
         try:
             # LOGGING
             kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception)
-            verbose_router_logger.info(f"Falling back to model_group = {mg}")
+            verbose_router_logger.info(f"Falling back to model_group = {mask_sensitive_structure(mg)}")
             if isinstance(mg, str):
                 kwargs["model"] = mg
             elif isinstance(mg, dict):

tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py

@@ -197,3 +197,46 @@ def test_cost_per_token_fields_not_masked():
     # Actual secrets must still be masked
     assert "*" in masked["api_key"]
     assert "*" in masked["access_token"]
+
+
+def test_mask_sensitive_structure_passes_through_plain_topology_names():
+    """Fallback groups are usually lists of model-group name strings; those
+    carry no secrets and must survive verbatim so opt-in debug output stays useful."""
+    from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
+
+    assert mask_sensitive_structure(["gpt-4", "claude-3-haiku"]) == ["gpt-4", "claude-3-haiku"]
+    assert mask_sensitive_structure([{"gpt-3.5-turbo": ["claude-3-haiku"]}]) == [
+        {"gpt-3.5-turbo": ["claude-3-haiku"]}
+    ]
+    assert mask_sensitive_structure(None) is None
+
+
+def test_mask_sensitive_structure_masks_credentials_in_inline_fallback_dicts():
+    """An inline-dict fallback can carry provider credentials; those values must be
+    masked before the structure is embedded in a client-facing error message."""
+    from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
+
+    secret = "sk-INLINEFALLBACKSECRET1234567890"
+    aws_secret = "wJalrXUtnFEMIK7MDENGbPxRfiCYSECRETKEY"
+    masked = mask_sensitive_structure(
+        [{"model": "openai/gpt-4", "api_key": secret, "aws_secret_access_key": aws_secret}]
+    )
+
+    rendered = str(masked)
+    assert secret not in rendered
+    assert aws_secret not in rendered
+    # Non-secret keys stay visible so the fallback wiring remains debuggable
+    assert masked[0]["model"] == "openai/gpt-4"
+    assert "*" in masked[0]["api_key"]
+    assert "*" in masked[0]["aws_secret_access_key"]
+
+
+def test_mask_sensitive_structure_masks_credentials_nested_in_config_shape():
+    """Credentials nested inside the {group: [fallbacks]} config shape must also be masked."""
+    from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
+
+    secret = "sk-NESTEDINLINESECRET0987654321"
+    masked = mask_sensitive_structure(
+        [{"primary-group": [{"model": "gpt-4o", "api_key": secret}]}]
+    )
+    assert secret not in str(masked)

tests/test_litellm/test_router_exception_redaction.py

@@ -5,15 +5,21 @@ The Router historically appended internal config names (model_group,
 fallback_model_group, fallback failure detail, deployment timeouts,
 context_window_fallbacks dict, etc.) onto the message of the exception
 it re-raises. That message is then surfaced to clients by
-ProxyException, leaking the proxy's internal wiring.
+ProxyException, leaking the proxy's internal wiring and, when fallbacks
+are configured as inline deployment dicts, the provider credentials
+inside those dicts.
 
 The flag defaults to True to preserve historical behavior (no
 breaking change for existing deployments). Set it to False to redact
-those strings from the raised exception's message.
+those strings from the raised exception's message. Regardless of the
+flag, provider credentials inside inline-dict fallbacks are now masked
+so a raw api_key / aws_* value never reaches the client.
 
 These tests verify that with the flag ON (default) the historical
-leak strings appear in the raised exception's message, and with the
-flag OFF the proxy's internal wiring is redacted.
+topology strings appear in the raised exception's message, with the
+flag OFF the proxy's internal wiring is redacted, and that a raw
+provider credential never appears in the message regardless of the
+flag.
 
 Five leak sites are gated in `litellm/router.py`:
 
@@ -40,6 +46,7 @@ _RECEIVED_MODEL_GROUP_PHRASE = "Received Model Group="
 _AVAILABLE_FALLBACKS_PHRASE = "Available Model Group Fallbacks="
 _CONTEXT_WINDOW_HINT_PHRASE = "context_window_fallbacks="
 _INTERNAL_MODEL_GROUP_NAME = "all-anthropic/claude-secret-internal"
+_FALLBACK_CREDENTIAL = "sk-INLINEFALLBACKSECRET1234567890"
 
 
 def _router_with_rate_limit_failure() -> Router:
@@ -76,6 +83,37 @@ def _router_with_context_window_failure() -> Router:
     )
 
 
+def _router_with_credentialed_fallback() -> Router:
+    """Primary fails, and its fallback is an inline dict that carries a provider
+    api_key. When the fallback also fails, the router embeds that dict in the
+    exception message, which is where a raw credential would otherwise leak."""
+    return Router(
+        model_list=[
+            {
+                "model_name": _INTERNAL_MODEL_GROUP_NAME,
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_key": "key",
+                    "mock_response": "litellm.RateLimitError",
+                },
+                "model_info": {"id": "secret-deployment-id"},
+            },
+        ],
+        fallbacks=[
+            {
+                _INTERNAL_MODEL_GROUP_NAME: [
+                    {
+                        "model": "gpt-4o",
+                        "api_key": _FALLBACK_CREDENTIAL,
+                        "mock_response": "litellm.RateLimitError",
+                    }
+                ]
+            }
+        ],
+        num_retries=0,
+    )
+
+
 @pytest.fixture(autouse=True)
 def _reset_expose_flag():
     """Each test starts with the flag in its default (on) state."""
@@ -110,7 +148,8 @@ async def test_flag_off_does_not_leak_received_model_group():
 
 
 @pytest.mark.asyncio
-async def test_default_leaks_received_model_group():
+async def test_flag_on_shows_received_model_group():
+    litellm.expose_router_debug_in_errors = True
     router = _router_with_rate_limit_failure()
     with pytest.raises(litellm.RateLimitError) as excinfo:
         await router.acompletion(
@@ -142,7 +181,8 @@ async def test_flag_off_does_not_leak_context_window_fallback_hint():
 
 
 @pytest.mark.asyncio
-async def test_default_leaks_context_window_fallback_hint():
+async def test_flag_on_shows_context_window_fallback_hint():
+    litellm.expose_router_debug_in_errors = True
     router = _router_with_context_window_failure()
     with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
         await router.acompletion(
@@ -153,7 +193,7 @@ async def test_default_leaks_context_window_fallback_hint():
     assert _CONTEXT_WINDOW_HINT_PHRASE in msg, msg
     # Site 5 also fires for ContextWindow errors that exit the
     # orchestrator without fallback resolution, so the model_group
-    # name leaks under the default behavior.
+    # name is shown under the opt-in behavior.
     assert _INTERNAL_MODEL_GROUP_NAME in msg, msg
 
 
@@ -192,7 +232,8 @@ async def test_flag_off_does_not_leak_when_no_fallback_group_found():
 
 
 @pytest.mark.asyncio
-async def test_default_leaks_when_no_fallback_group_found():
+async def test_flag_on_shows_when_no_fallback_group_found():
+    litellm.expose_router_debug_in_errors = True
     router = Router(
         model_list=[
             {
@@ -258,7 +299,8 @@ async def test_flag_off_does_not_leak_deployment_timeout_debug():
 
 
 @pytest.mark.asyncio
-async def test_default_leaks_deployment_timeout_debug():
+async def test_flag_on_shows_deployment_timeout_debug():
+    litellm.expose_router_debug_in_errors = True
     router = _router_with_plain_deployment()
     with pytest.raises(litellm.Timeout) as excinfo:
         await router.acompletion(
@@ -298,7 +340,8 @@ async def test_flag_off_does_not_leak_content_policy_fallback_hint():
 
 
 @pytest.mark.asyncio
-async def test_default_leaks_content_policy_fallback_hint():
+async def test_flag_on_shows_content_policy_fallback_hint():
+    litellm.expose_router_debug_in_errors = True
     router = _router_with_plain_deployment()
     with pytest.raises(litellm.ContentPolicyViolationError) as excinfo:
         await router.acompletion(
@@ -309,3 +352,82 @@ async def test_default_leaks_content_policy_fallback_hint():
     msg = excinfo.value.message
     assert "content_policy_fallback=" in msg, msg
     assert _INTERNAL_MODEL_GROUP_NAME in msg, msg
+
+
+# --- Credential masking: raw provider keys never leak, either flag state ----
+
+
+@pytest.mark.asyncio
+async def test_flag_off_hides_fallback_credentials():
+    litellm.expose_router_debug_in_errors = False
+    router = _router_with_credentialed_fallback()
+    with pytest.raises(litellm.RateLimitError) as excinfo:
+        await router.acompletion(
+            model=_INTERNAL_MODEL_GROUP_NAME,
+            messages=[{"role": "user", "content": "hi"}],
+        )
+    msg = excinfo.value.message
+    assert _FALLBACK_CREDENTIAL not in msg, msg
+    assert _AVAILABLE_FALLBACKS_PHRASE not in msg, msg
+
+
+@pytest.mark.asyncio
+async def test_flag_on_masks_fallback_credentials():
+    litellm.expose_router_debug_in_errors = True
+    router = _router_with_credentialed_fallback()
+    with pytest.raises(litellm.RateLimitError) as excinfo:
+        await router.acompletion(
+            model=_INTERNAL_MODEL_GROUP_NAME,
+            messages=[{"role": "user", "content": "hi"}],
+        )
+    msg = excinfo.value.message
+    # The raw credential must never appear, even though debug exposure is on
+    assert _FALLBACK_CREDENTIAL not in msg, msg
+    # The fallback wiring is still shown (masking preserves structure, it does
+    # not drop the whole message), so the api_key key name survives
+    assert "api_key" in msg, msg
+
+
+@pytest.mark.asyncio
+async def test_flag_on_scrubs_credential_from_inner_fallback_exception_string():
+    """If the fallback attempt itself raises an exception whose message embeds a
+    raw provider credential (e.g. a provider SDK echoing back the api_key it was
+    called with), that string is re-embedded via `Error doing the fallback: ...`
+    on the terminal raise. The router must scrub known secret patterns from it.
+    The primary fails with a benign rate-limit; the fallback deployment fails
+    with an exception whose text contains the secret."""
+    litellm.expose_router_debug_in_errors = True
+    inner_secret = "sk-INNERFALLBACKEXCEPTIONSECRET1234"
+    router = Router(
+        model_list=[
+            {
+                "model_name": _INTERNAL_MODEL_GROUP_NAME,
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_key": "key",
+                    "mock_response": "litellm.RateLimitError",
+                },
+                "model_info": {"id": "secret-deployment-id"},
+            },
+            {
+                "model_name": "fallback-group",
+                "litellm_params": {
+                    "model": "gpt-4o",
+                    "api_key": "key",
+                    "mock_response": f"Exception: content_filter_policy - api_key={inner_secret}",
+                },
+                "model_info": {"id": "fallback-deployment-id"},
+            },
+        ],
+        fallbacks=[{_INTERNAL_MODEL_GROUP_NAME: ["fallback-group"]}],
+        num_retries=0,
+    )
+    with pytest.raises(litellm.RateLimitError) as excinfo:
+        await router.acompletion(
+            model=_INTERNAL_MODEL_GROUP_NAME,
+            messages=[{"role": "user", "content": "hi"}],
+        )
+    msg = excinfo.value.message
+    assert "Error doing the fallback:" in msg, msg
+    assert inner_secret not in msg, msg
+    assert "REDACTED" in msg, msg