Merge pull request #26484 from stuxf/fix/master-key-pass-the-hash

chore(auth): substitute alias for master key on UserAPIKeyAuth

litellm/constants.py

@@ -1393,6 +1393,10 @@ except (ValueError, TypeError):
 LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
 LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
 LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"
+# Stable identifier substituted in place of the master key on UserAPIKeyAuth
+# objects so the master key (or its hash) never propagates to spend logs,
+# Prometheus metrics, audit trails, or any other downstream consumer.
+LITELLM_PROXY_MASTER_KEY_ALIAS = "litellm_proxy_master_key"
 
 # Key Rotation Constants
 LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")

litellm/proxy/auth/user_api_key_auth.py

@@ -21,6 +21,7 @@ import litellm
 from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm._service_logger import ServiceLogging
 from litellm.caching import DualCache
+from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
 from litellm.proxy._types import *
@@ -1119,10 +1120,14 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
             )
 
         if is_master_key_valid:
+            # Substitute a stable alias for the raw master key so neither the
+            # master key nor its hash propagates into spend logs, Prometheus
+            # /metrics labels, audit trails, rate-limit buckets, or any other
+            # downstream consumer of UserAPIKeyAuth.api_key.
             _user_api_key_obj = await _return_user_api_key_auth_obj(
                 user_obj=None,
                 user_role=LitellmUserRoles.PROXY_ADMIN,
-                api_key=master_key,
+                api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,
                 parent_otel_span=parent_otel_span,
                 valid_token_dict={
                     **end_user_params,

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -53,20 +53,13 @@ def _get_max_string_length_prompt_in_db() -> int:
 
 
 def _is_master_key(api_key: Optional[str], _master_key: Optional[str]) -> bool:
+    """
+    Raw-only constant-time master-key comparison. The hashed form is never
+    considered equivalent — only the raw master-key string matches.
+    """
     if _master_key is None or api_key is None:
         return False
-
-    ## string comparison
-    is_master_key = secrets.compare_digest(api_key, _master_key)
-    if is_master_key:
-        return True
-
-    ## hash comparison
-    is_master_key = secrets.compare_digest(api_key, hash_token(_master_key))
-    if is_master_key:
-        return True
-
-    return False
+    return secrets.compare_digest(api_key, _master_key)
 
 
 def _get_spend_logs_metadata(
@@ -235,8 +228,6 @@ def _extract_usage_for_ocr_call(response_obj: Any, response_obj_dict: dict) -> d
 def get_logging_payload(  # noqa: PLR0915
     kwargs, response_obj, start_time, end_time
 ) -> SpendLogsPayload:
-    from litellm.proxy.proxy_server import general_settings, master_key
-
     if kwargs is None:
         kwargs = {}
 
@@ -295,11 +286,6 @@ def get_logging_payload(  # noqa: PLR0915
         if api_key.startswith("sk-"):
             # hash the api_key
             api_key = hash_token(api_key)
-        if (
-            _is_master_key(api_key=api_key, _master_key=master_key)
-            and general_settings.get("disable_adding_master_key_hash_to_db") is True
-        ):
-            api_key = "litellm_proxy_master_key"  # use a known alias, if the user disabled storing master key in db
 
     if (
         standard_logging_payload is not None
@@ -324,11 +310,6 @@ def get_logging_payload(  # noqa: PLR0915
         and standard_logging_payload.get("request_tags") is not None
     ):  # use 'tags' from standard logging payload instead
         request_tags = json.dumps(standard_logging_payload["request_tags"])
-    if (
-        _is_master_key(api_key=api_key, _master_key=master_key)
-        and general_settings.get("disable_adding_master_key_hash_to_db") is True
-    ):
-        api_key = "litellm_proxy_master_key"  # use a known alias, if the user disabled storing master key in db
 
     _model_id = metadata.get("model_info", {}).get("id", "")
     _model_group = metadata.get("model_group", "")

tests/proxy_unit_tests/test_key_generate_prisma.py

@@ -2715,7 +2715,12 @@ async def test_master_key_hashing(prisma_client):
             request=request, api_key=bearer_token
         )
 
-        assert result.api_key == hash_token(master_key)
+        # Master-key auth substitutes a stable alias so the master key (or
+        # its hash) never propagates into spend logs / metrics / audit trails.
+        from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
+
+        assert result.api_key == LITELLM_PROXY_MASTER_KEY_ALIAS
+        assert result.api_key != hash_token(master_key)
 
     except Exception as e:
         print("Got Exception", e)

tests/proxy_unit_tests/test_user_api_key_auth.py

@@ -1118,11 +1118,17 @@ async def test_jwt_non_admin_team_route_access(monkeypatch):
 @pytest.mark.asyncio
 async def test_x_litellm_api_key():
     """
-    Check if auth can pick up x-litellm-api-key header, even if Bearer token is provided
+    Check if auth can pick up x-litellm-api-key header, even if Bearer token is provided.
+
+    On a master-key match, ``UserAPIKeyAuth.api_key`` (and the derived
+    ``token``) are now the stable alias ``LITELLM_PROXY_MASTER_KEY_ALIAS``
+    rather than ``hash_token(master_key)`` — the master key (or its hash)
+    must not propagate into spend logs / metrics / audit trails.
     """
     from fastapi import Request
     from starlette.datastructures import URL
 
+    from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
     from litellm.proxy._types import (
         LiteLLM_TeamTable,
         LiteLLM_TeamTableCachedObj,
@@ -1148,7 +1154,8 @@ async def test_x_litellm_api_key():
         api_key="Bearer " + ignored_key,
         custom_litellm_key_header=master_key,
     )
-    assert valid_token.token == hash_token(master_key)
+    assert valid_token.token == LITELLM_PROXY_MASTER_KEY_ALIAS
+    assert valid_token.token != hash_token(master_key)
 
 
 @pytest.mark.asyncio

tests/test_litellm/proxy/auth/test_user_api_key_auth.py

@@ -2581,3 +2581,49 @@ async def test_centralized_common_checks_user_http_exception_isolates_to_user_on
     finally:
         for k, v in originals.items():
             setattr(_proxy_server_mod, k, v)
+
+
+@pytest.mark.asyncio
+async def test_master_key_auth_substitutes_alias_for_api_key():
+    """
+    When the master key authenticates a request, the resulting
+    ``UserAPIKeyAuth.api_key`` must be the stable alias
+    ``LITELLM_PROXY_MASTER_KEY_ALIAS`` — never the raw master key (which
+    would propagate downstream and be hashed into spend logs, Prometheus
+    ``/metrics`` labels, or audit trails) and never the master-key hash.
+    """
+    from fastapi import Request
+    from starlette.datastructures import URL
+
+    from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
+    from litellm.proxy.auth.user_api_key_auth import _user_api_key_auth_builder
+    from litellm.proxy.utils import hash_token
+
+    import litellm.proxy.proxy_server as _proxy_server_mod
+
+    attrs = _proxy_server_attrs_for_custom_auth(user_custom_auth=None)
+    master_key = attrs["master_key"]
+    _orig = {k: getattr(_proxy_server_mod, k, None) for k in attrs}
+    try:
+        for k, v in attrs.items():
+            setattr(_proxy_server_mod, k, v)
+
+        request = Request(scope={"type": "http"})
+        request._url = URL(url="/chat/completions")
+
+        result = await _user_api_key_auth_builder(
+            request=request,
+            api_key=f"Bearer {master_key}",
+            azure_api_key_header="",
+            anthropic_api_key_header=None,
+            google_ai_studio_api_key_header=None,
+            azure_apim_header=None,
+            request_data={},
+        )
+
+        assert result.api_key == LITELLM_PROXY_MASTER_KEY_ALIAS
+        assert result.api_key != master_key
+        assert result.api_key != hash_token(master_key)
+    finally:
+        for k, v in _orig.items():
+            setattr(_proxy_server_mod, k, v)

tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py

@@ -1513,9 +1513,13 @@ class TestIsMasterKey:
     def test_non_matching_key_returns_false(self):
         assert _is_master_key(api_key="sk-other", _master_key="sk-master") is False
 
-    def test_hashed_key_returns_true(self):
+    def test_master_key_hash_is_rejected(self):
+        """
+        ``_is_master_key`` must not accept ``hash_token(master_key)`` as
+        equivalent to the raw master key — only the raw value matches.
+        """
         from litellm.proxy.utils import hash_token
 
         master = "sk-master-key-123"
         hashed = hash_token(master)
-        assert _is_master_key(api_key=hashed, _master_key=master) is True
+        assert _is_master_key(api_key=hashed, _master_key=master) is False