fix(logging): hash raw credentials landing in user_api_key_hash

user_api_key_hash is copied verbatim from input metadata into the standard logging payload, so a raw sk- virtual key or JWT reaching that field is persisted in cleartext by every durable sink (spend logs, S3 request logs, Prometheus labels). Documented custom auth returns the raw client credential, and a proxy without a master key does the same, so this is reachable without any SDK misuse.

Sanitize at both the key-metadata source and the two standard-logging-metadata builders: sk- keys (with or without a Bearer prefix) and JWTs are replaced with their sha256 digest, the digest virtual keys are already stored under, while existing digests and non-credential identifiers such as the master key alias pass through so downstream grouping stays stable.

litellm/litellm_core_utils/credential_hashing.py

@@ -0,0 +1,45 @@
+import base64
+import hashlib
+import json
+import re
+
+_SHA256_HEX_RE = re.compile(r"[a-fA-F0-9]{64}")
+_BEARER_PREFIX = "bearer "
+
+
+def is_valid_sha256_hash(value: str) -> bool:
+    return bool(_SHA256_HEX_RE.fullmatch(value))
+
+
+def _is_jwt(value: str) -> bool:
+    header, _, rest = value.partition(".")
+    payload, _, signature = rest.partition(".")
+    if not header or not payload or not signature or "." in signature:
+        return False
+    try:
+        decoded_header = json.loads(base64.urlsafe_b64decode(header + "=" * (-len(header) % 4)))
+    except Exception:
+        return False
+    return isinstance(decoded_header, dict) and "alg" in decoded_header
+
+
+def sanitize_key_hash(value: str | None) -> str | None:
+    """
+    Guarantee that a value destined for a `user_api_key_hash` field carries no
+    credential material, since those fields land in durable sinks (spend logs,
+    S3 request logs, Prometheus labels).
+
+    Virtual keys (`sk-...`, optionally still `Bearer `-prefixed) and JWTs are
+    replaced with their sha256 digest, matching the digest virtual keys are
+    already stored and looked up under. Values that are already a digest, and
+    non-credential identifiers such as the master key alias, pass through
+    untouched so downstream grouping stays stable.
+    """
+    if value is None:
+        return None
+    credential = value[len(_BEARER_PREFIX) :] if value[: len(_BEARER_PREFIX)].lower() == _BEARER_PREFIX else value
+    if is_valid_sha256_hash(credential):
+        return credential
+    if credential.startswith("sk-") or _is_jwt(credential):
+        return hashlib.sha256(credential.encode()).hexdigest()
+    return value

litellm/litellm_core_utils/litellm_logging.py

@@ -5,7 +5,6 @@ import copy
 import datetime
 import json
 import os
-import re
 import subprocess
 import sys
 import time
@@ -65,6 +64,10 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger
 from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
 from litellm.litellm_core_utils.core_helpers import reconstruct_model_name
+from litellm.litellm_core_utils.credential_hashing import (
+    is_valid_sha256_hash,
+    sanitize_key_hash,
+)
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
@@ -4508,11 +4511,6 @@ def use_custom_pricing_for_model(litellm_params: Optional[dict]) -> bool:
     return False
 
 
-def is_valid_sha256_hash(value: str) -> bool:
-    # Check if the value is a valid SHA-256 hash (64 hexadecimal characters)
-    return bool(re.fullmatch(r"[a-fA-F0-9]{64}", value))
-
-
 class StandardLoggingPayloadSetup:
     @staticmethod
     def cleanup_timestamps(
@@ -4694,6 +4692,7 @@ class StandardLoggingPayloadSetup:
             user_api_key = metadata.get("user_api_key")
             if user_api_key and isinstance(user_api_key, str) and is_valid_sha256_hash(user_api_key):
                 clean_metadata["user_api_key_hash"] = user_api_key
+            clean_metadata["user_api_key_hash"] = sanitize_key_hash(clean_metadata["user_api_key_hash"])
             _potential_requester_metadata = metadata.get(
                 "metadata", None
             )  # check if user passed metadata in the sdk request - e.g. metadata for langsmith logging - https://docs.litellm.ai/docs/observability/langsmith_integration#set-langsmith-fields
@@ -5536,6 +5535,7 @@ def get_standard_logging_metadata(
         if metadata.get("user_api_key") is not None:
             if is_valid_sha256_hash(str(metadata.get("user_api_key"))):
                 clean_metadata["user_api_key_hash"] = metadata.get("user_api_key")  # this is the hash
+        clean_metadata["user_api_key_hash"] = sanitize_key_hash(clean_metadata["user_api_key_hash"])
     return clean_metadata
 
 

litellm/proxy/litellm_pre_call_utils.py

@@ -15,6 +15,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm._service_logger import ServiceLogging
 from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS, PRE_CALL_EXECUTED_GUARDRAILS_KEY
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+from litellm.litellm_core_utils.credential_hashing import sanitize_key_hash
 from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
     iter_client_callback_metadata_dicts,
 )
@@ -1011,7 +1012,7 @@ class LiteLLMProxyRequestSetup:
         user_api_key_dict: UserAPIKeyAuth,
     ) -> StandardLoggingUserAPIKeyMetadata:
         user_api_key_logged_metadata = StandardLoggingUserAPIKeyMetadata(
-            user_api_key_hash=user_api_key_dict.api_key,  # just the hashed token
+            user_api_key_hash=sanitize_key_hash(user_api_key_dict.api_key),
             user_api_key_alias=user_api_key_dict.key_alias,
             user_api_key_spend=user_api_key_dict.spend,
             user_api_key_max_budget=user_api_key_dict.max_budget,

tests/test_litellm/litellm_core_utils/test_credential_hashing.py

@@ -0,0 +1,51 @@
+import base64
+import hashlib
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath("../../.."))
+
+from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
+from litellm.litellm_core_utils.credential_hashing import sanitize_key_hash
+
+
+def _jwt(header: dict) -> str:
+    def segment(payload: dict) -> str:
+        return base64.urlsafe_b64encode(json.dumps(payload).encode()).rstrip(b"=").decode()
+
+    return f"{segment(header)}.{segment({'sub': 'user-1'})}.signature"
+
+
+def test_raw_virtual_key_is_hashed():
+    assert sanitize_key_hash("sk-1234") == hashlib.sha256(b"sk-1234").hexdigest()
+
+
+def test_bearer_prefixed_virtual_key_is_hashed_without_prefix():
+    assert sanitize_key_hash("Bearer sk-1234") == hashlib.sha256(b"sk-1234").hexdigest()
+
+
+def test_jwt_is_hashed():
+    token = _jwt({"alg": "RS256", "typ": "JWT"})
+    assert sanitize_key_hash(token) == hashlib.sha256(token.encode()).hexdigest()
+
+
+def test_existing_hash_passes_through():
+    digest = hashlib.sha256(b"sk-1234").hexdigest()
+    assert sanitize_key_hash(digest) == digest
+
+
+def test_master_key_alias_passes_through():
+    assert sanitize_key_hash(LITELLM_PROXY_MASTER_KEY_ALIAS) == LITELLM_PROXY_MASTER_KEY_ALIAS
+
+
+def test_non_credential_identifier_passes_through():
+    assert sanitize_key_hash("test_hash") == "test_hash"
+
+
+def test_dotted_identifier_is_not_treated_as_jwt():
+    assert sanitize_key_hash("team.project.key") == "team.project.key"
+
+
+def test_none_passes_through():
+    assert sanitize_key_hash(None) is None

tests/test_litellm/litellm_core_utils/test_litellm_logging.py

@@ -4101,3 +4101,41 @@ def test_pre_call_does_not_pin_request_in_module_state(logging_obj):
     logging_obj.post_call(original_response='{"ok": true}', input=big_input, api_key="sk-test")
 
     assert litellm.error_logs == {}
+
+
+def test_standard_logging_metadata_hashes_raw_virtual_key_in_key_hash_field():
+    """
+    user_api_key_hash lands in durable sinks (spend logs, S3 request logs). A raw
+    virtual key arriving in that field must never be persisted verbatim
+    """
+    import hashlib
+
+    from litellm.litellm_core_utils.litellm_logging import (
+        StandardLoggingPayloadSetup,
+        get_standard_logging_metadata,
+    )
+
+    raw_key = "sk-my-secret-virtual-key"
+    expected = hashlib.sha256(raw_key.encode()).hexdigest()
+
+    setup_result = StandardLoggingPayloadSetup.get_standard_logging_metadata(
+        {"user_api_key_hash": raw_key}
+    )
+    legacy_result = get_standard_logging_metadata({"user_api_key_hash": raw_key})
+
+    assert setup_result["user_api_key_hash"] == expected
+    assert legacy_result["user_api_key_hash"] == expected
+
+
+def test_standard_logging_metadata_preserves_hashed_key():
+    import hashlib
+
+    from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+
+    hashed_key = hashlib.sha256(b"sk-my-secret-virtual-key").hexdigest()
+
+    result = StandardLoggingPayloadSetup.get_standard_logging_metadata(
+        {"user_api_key_hash": hashed_key}
+    )
+
+    assert result["user_api_key_hash"] == hashed_key

tests/test_litellm/proxy/test_litellm_pre_call_utils.py

@@ -5421,3 +5421,36 @@ async def test_overwrite_user_with_key_hash_rejects_alias_without_marker(monkeyp
     )
 
     assert updated_data["user"] == "caller-chosen-id"
+
+
+def test_get_sanitized_user_information_from_key_hashes_raw_credential():
+    """
+    Custom auth (documented as `return UserAPIKeyAuth(api_key=api_key)`) and
+    proxies running without a master key hand back the raw client credential.
+    It must be hashed before it reaches logging metadata, while proxy-validated
+    values (key hashes, the master key alias) stay untouched
+    """
+    import hashlib
+
+    from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
+
+    raw_key = "sk-my-secret-virtual-key"
+
+    sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(
+        user_api_key_dict=UserAPIKeyAuth(api_key=raw_key)
+    )
+    assert sanitized["user_api_key_hash"] == hashlib.sha256(raw_key.encode()).hexdigest()
+
+    key_hash = hashlib.sha256(raw_key.encode()).hexdigest()
+    assert (
+        LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(
+            user_api_key_dict=UserAPIKeyAuth(api_key=key_hash)
+        )["user_api_key_hash"]
+        == key_hash
+    )
+    assert (
+        LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(
+            user_api_key_dict=UserAPIKeyAuth(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS)
+        )["user_api_key_hash"]
+        == LITELLM_PROXY_MASTER_KEY_ALIAS
+    )