From ca29df943d991f8827c03fbc42af52d7ce9268cf Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:18:09 +0000 Subject: [PATCH] fix(logging): hash raw credentials landing in user_api_key_hash user_api_key_hash is copied verbatim from input metadata into the standard logging payload, so a raw sk- virtual key or JWT reaching that field is persisted in cleartext by every durable sink (spend logs, S3 request logs, Prometheus labels). Documented custom auth returns the raw client credential, and a proxy without a master key does the same, so this is reachable without any SDK misuse. Sanitize at both the key-metadata source and the two standard-logging-metadata builders: sk- keys (with or without a Bearer prefix) and JWTs are replaced with their sha256 digest, the digest virtual keys are already stored under, while existing digests and non-credential identifiers such as the master key alias pass through so downstream grouping stays stable. --- .../litellm_core_utils/credential_hashing.py | 45 ++++++++++++++++ litellm/litellm_core_utils/litellm_logging.py | 12 ++--- litellm/proxy/litellm_pre_call_utils.py | 3 +- .../test_credential_hashing.py | 51 +++++++++++++++++++ .../test_litellm_logging.py | 38 ++++++++++++++ .../proxy/test_litellm_pre_call_utils.py | 33 ++++++++++++ 6 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 litellm/litellm_core_utils/credential_hashing.py create mode 100644 tests/test_litellm/litellm_core_utils/test_credential_hashing.py diff --git a/litellm/litellm_core_utils/credential_hashing.py b/litellm/litellm_core_utils/credential_hashing.py new file mode 100644 index 00000000000..204e19b5017 --- /dev/null +++ b/litellm/litellm_core_utils/credential_hashing.py @@ -0,0 +1,45 @@ +import base64 +import hashlib +import json +import re + +_SHA256_HEX_RE = re.compile(r"[a-fA-F0-9]{64}") +_BEARER_PREFIX = "bearer " + + +def is_valid_sha256_hash(value: str) -> bool: + return bool(_SHA256_HEX_RE.fullmatch(value)) + + +def _is_jwt(value: str) -> bool: + header, _, rest = value.partition(".") + payload, _, signature = rest.partition(".") + if not header or not payload or not signature or "." in signature: + return False + try: + decoded_header = json.loads(base64.urlsafe_b64decode(header + "=" * (-len(header) % 4))) + except Exception: + return False + return isinstance(decoded_header, dict) and "alg" in decoded_header + + +def sanitize_key_hash(value: str | None) -> str | None: + """ + Guarantee that a value destined for a `user_api_key_hash` field carries no + credential material, since those fields land in durable sinks (spend logs, + S3 request logs, Prometheus labels). + + Virtual keys (`sk-...`, optionally still `Bearer `-prefixed) and JWTs are + replaced with their sha256 digest, matching the digest virtual keys are + already stored and looked up under. Values that are already a digest, and + non-credential identifiers such as the master key alias, pass through + untouched so downstream grouping stays stable. + """ + if value is None: + return None + credential = value[len(_BEARER_PREFIX) :] if value[: len(_BEARER_PREFIX)].lower() == _BEARER_PREFIX else value + if is_valid_sha256_hash(credential): + return credential + if credential.startswith("sk-") or _is_jwt(credential): + return hashlib.sha256(credential.encode()).hexdigest() + return value diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e1e5499d008..af1f311b97c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5,7 +5,6 @@ import copy import datetime import json import os -import re import subprocess import sys import time @@ -65,6 +64,10 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger from litellm.litellm_core_utils.core_helpers import reconstruct_model_name +from litellm.litellm_core_utils.credential_hashing import ( + is_valid_sha256_hash, + sanitize_key_hash, +) from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, @@ -4508,11 +4511,6 @@ def use_custom_pricing_for_model(litellm_params: Optional[dict]) -> bool: return False -def is_valid_sha256_hash(value: str) -> bool: - # Check if the value is a valid SHA-256 hash (64 hexadecimal characters) - return bool(re.fullmatch(r"[a-fA-F0-9]{64}", value)) - - class StandardLoggingPayloadSetup: @staticmethod def cleanup_timestamps( @@ -4694,6 +4692,7 @@ class StandardLoggingPayloadSetup: user_api_key = metadata.get("user_api_key") if user_api_key and isinstance(user_api_key, str) and is_valid_sha256_hash(user_api_key): clean_metadata["user_api_key_hash"] = user_api_key + clean_metadata["user_api_key_hash"] = sanitize_key_hash(clean_metadata["user_api_key_hash"]) _potential_requester_metadata = metadata.get( "metadata", None ) # check if user passed metadata in the sdk request - e.g. metadata for langsmith logging - https://docs.litellm.ai/docs/observability/langsmith_integration#set-langsmith-fields @@ -5536,6 +5535,7 @@ def get_standard_logging_metadata( if metadata.get("user_api_key") is not None: if is_valid_sha256_hash(str(metadata.get("user_api_key"))): clean_metadata["user_api_key_hash"] = metadata.get("user_api_key") # this is the hash + clean_metadata["user_api_key_hash"] = sanitize_key_hash(clean_metadata["user_api_key_hash"]) return clean_metadata diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index a4cc4a62009..cc151f944b0 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -15,6 +15,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS, PRE_CALL_EXECUTED_GUARDRAILS_KEY from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.credential_hashing import sanitize_key_hash from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( iter_client_callback_metadata_dicts, ) @@ -1011,7 +1012,7 @@ class LiteLLMProxyRequestSetup: user_api_key_dict: UserAPIKeyAuth, ) -> StandardLoggingUserAPIKeyMetadata: user_api_key_logged_metadata = StandardLoggingUserAPIKeyMetadata( - user_api_key_hash=user_api_key_dict.api_key, # just the hashed token + user_api_key_hash=sanitize_key_hash(user_api_key_dict.api_key), user_api_key_alias=user_api_key_dict.key_alias, user_api_key_spend=user_api_key_dict.spend, user_api_key_max_budget=user_api_key_dict.max_budget, diff --git a/tests/test_litellm/litellm_core_utils/test_credential_hashing.py b/tests/test_litellm/litellm_core_utils/test_credential_hashing.py new file mode 100644 index 00000000000..08255598990 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_credential_hashing.py @@ -0,0 +1,51 @@ +import base64 +import hashlib +import json +import os +import sys + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS +from litellm.litellm_core_utils.credential_hashing import sanitize_key_hash + + +def _jwt(header: dict) -> str: + def segment(payload: dict) -> str: + return base64.urlsafe_b64encode(json.dumps(payload).encode()).rstrip(b"=").decode() + + return f"{segment(header)}.{segment({'sub': 'user-1'})}.signature" + + +def test_raw_virtual_key_is_hashed(): + assert sanitize_key_hash("sk-1234") == hashlib.sha256(b"sk-1234").hexdigest() + + +def test_bearer_prefixed_virtual_key_is_hashed_without_prefix(): + assert sanitize_key_hash("Bearer sk-1234") == hashlib.sha256(b"sk-1234").hexdigest() + + +def test_jwt_is_hashed(): + token = _jwt({"alg": "RS256", "typ": "JWT"}) + assert sanitize_key_hash(token) == hashlib.sha256(token.encode()).hexdigest() + + +def test_existing_hash_passes_through(): + digest = hashlib.sha256(b"sk-1234").hexdigest() + assert sanitize_key_hash(digest) == digest + + +def test_master_key_alias_passes_through(): + assert sanitize_key_hash(LITELLM_PROXY_MASTER_KEY_ALIAS) == LITELLM_PROXY_MASTER_KEY_ALIAS + + +def test_non_credential_identifier_passes_through(): + assert sanitize_key_hash("test_hash") == "test_hash" + + +def test_dotted_identifier_is_not_treated_as_jwt(): + assert sanitize_key_hash("team.project.key") == "team.project.key" + + +def test_none_passes_through(): + assert sanitize_key_hash(None) is None diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index edc257f4c3f..060e65587ac 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -4101,3 +4101,41 @@ def test_pre_call_does_not_pin_request_in_module_state(logging_obj): logging_obj.post_call(original_response='{"ok": true}', input=big_input, api_key="sk-test") assert litellm.error_logs == {} + + +def test_standard_logging_metadata_hashes_raw_virtual_key_in_key_hash_field(): + """ + user_api_key_hash lands in durable sinks (spend logs, S3 request logs). A raw + virtual key arriving in that field must never be persisted verbatim + """ + import hashlib + + from litellm.litellm_core_utils.litellm_logging import ( + StandardLoggingPayloadSetup, + get_standard_logging_metadata, + ) + + raw_key = "sk-my-secret-virtual-key" + expected = hashlib.sha256(raw_key.encode()).hexdigest() + + setup_result = StandardLoggingPayloadSetup.get_standard_logging_metadata( + {"user_api_key_hash": raw_key} + ) + legacy_result = get_standard_logging_metadata({"user_api_key_hash": raw_key}) + + assert setup_result["user_api_key_hash"] == expected + assert legacy_result["user_api_key_hash"] == expected + + +def test_standard_logging_metadata_preserves_hashed_key(): + import hashlib + + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + hashed_key = hashlib.sha256(b"sk-my-secret-virtual-key").hexdigest() + + result = StandardLoggingPayloadSetup.get_standard_logging_metadata( + {"user_api_key_hash": hashed_key} + ) + + assert result["user_api_key_hash"] == hashed_key diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 1437899f561..0da5ba5cc7f 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -5421,3 +5421,36 @@ async def test_overwrite_user_with_key_hash_rejects_alias_without_marker(monkeyp ) assert updated_data["user"] == "caller-chosen-id" + + +def test_get_sanitized_user_information_from_key_hashes_raw_credential(): + """ + Custom auth (documented as `return UserAPIKeyAuth(api_key=api_key)`) and + proxies running without a master key hand back the raw client credential. + It must be hashed before it reaches logging metadata, while proxy-validated + values (key hashes, the master key alias) stay untouched + """ + import hashlib + + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + raw_key = "sk-my-secret-virtual-key" + + sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=UserAPIKeyAuth(api_key=raw_key) + ) + assert sanitized["user_api_key_hash"] == hashlib.sha256(raw_key.encode()).hexdigest() + + key_hash = hashlib.sha256(raw_key.encode()).hexdigest() + assert ( + LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=UserAPIKeyAuth(api_key=key_hash) + )["user_api_key_hash"] + == key_hash + ) + assert ( + LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=UserAPIKeyAuth(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS) + )["user_api_key_hash"] + == LITELLM_PROXY_MASTER_KEY_ALIAS + )