From e2932d22c0b6d6e0a01c4346fe304525eed9bf8e Mon Sep 17 00:00:00 2001 From: Nitish Agarwal <1592163+nitishagar@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:06:36 +0530 Subject: [PATCH 1/5] fix(spend-tracking): hash raw api keys before persisting to spend logs (#30670) The Request Logs "Key Hash" column binds to the SpendLogs metadata user_api_key field, which _get_spend_logs_metadata copied verbatim from request metadata without hashing. The top-level api_key column only hashed values starting with sk-. So a non-sk- passthrough provider key, or a Bearer-prefixed key on paths that do not strip it, could be persisted and shown in plaintext. A single helper now redacts both fields at the SpendLogs builder: it strips a Bearer prefix, passes through values already a sha256 hash or hashed-jwt- identifier, and otherwise hashes with hash_token. sk- keys still map to the same canonical hash, so log and spend correlation is unchanged --- .../spend_tracking/spend_tracking_utils.py | 42 ++- .../test_spend_tracking_utils.py | 270 ++++++++++++++++-- 2 files changed, 290 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index aef06a3c668..e8aa2325a18 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, ) +from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error @@ -65,6 +66,24 @@ def _is_master_key(api_key: Optional[str], _master_key: Optional[str]) -> bool: return secrets.compare_digest(api_key, _master_key) +_HASHED_JWT_RE = re.compile(r"^hashed-jwt-[a-fA-F0-9]{64}$") + + +def _redact_logged_api_key( + value: str | None, *, already_hashed: bool = False +) -> str | None: + if not isinstance(value, str) or not value: + return None + stripped = re.sub(r"(?i)^bearer ", "", value) + if not stripped: + return None + if already_hashed and is_valid_sha256_hash(stripped): + return stripped + if _HASHED_JWT_RE.match(stripped): + return stripped + return hash_token(stripped) + + def _get_spend_logs_metadata( metadata: Optional[dict], applied_guardrails: Optional[List[str]] = None, @@ -121,6 +140,16 @@ def _get_spend_logs_metadata( key: metadata.get(key) for key in SpendLogsMetadata.__annotations__.keys() } ) + _raw_key = clean_metadata.get("user_api_key") + _trusted_hash = metadata.get("user_api_key_hash") if metadata else None + _already_hashed = ( + isinstance(_trusted_hash, str) + and is_valid_sha256_hash(_trusted_hash) + and _trusted_hash == _raw_key + ) + clean_metadata["user_api_key"] = _redact_logged_api_key( + _raw_key, already_hashed=_already_hashed + ) clean_metadata["applied_guardrails"] = applied_guardrails clean_metadata["batch_models"] = batch_models clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata @@ -283,10 +312,13 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs "completion_tokens", 0 ) standard_logging_total_tokens = standard_logging_payload.get("total_tokens", 0) - if api_key is not None and isinstance(api_key, str): - if api_key.startswith("sk-"): - # hash the api_key - api_key = hash_token(api_key) + _trusted_hash = metadata.get("user_api_key_hash") if metadata else None + _key_already_hashed = ( + isinstance(_trusted_hash, str) + and is_valid_sha256_hash(_trusted_hash) + and _trusted_hash == api_key + ) + api_key = _redact_logged_api_key(api_key, already_hashed=_key_already_hashed) or "" if ( standard_logging_payload is not None @@ -299,8 +331,6 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs end_user_id = end_user_id or standard_logging_payload["metadata"].get( "user_api_key_end_user_id" ) - # BUG FIX: Don't overwrite api_key when standard_logging_payload is None - # The api_key was already extracted from metadata (line 243) and hashed (lines 256-259) request_tags = ( safe_dumps(metadata.get("tags", [])) if isinstance(metadata.get("tags", []), list) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 0c7511589de..67d57d2d6bb 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -7,7 +7,6 @@ from datetime import timezone from typing import Any, cast import pytest -from fastapi.testclient import TestClient sys.path.insert( 0, os.path.abspath("../../../..") @@ -15,7 +14,6 @@ sys.path.insert( from unittest.mock import AsyncMock, MagicMock, patch -import litellm from litellm.constants import ( LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, @@ -30,12 +28,14 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_spend_logs_metadata, _get_vector_store_request_for_spend_logs_payload, _is_master_key, + _redact_logged_api_key, _redact_prompt_leaks_in_error_string, _sanitize_error_information_for_spend_logs, _sanitize_request_body_for_spend_logs_payload, _should_store_prompts_and_responses_in_spend_logs, get_logging_payload, ) +from litellm.proxy.utils import hash_token from litellm.types.utils import ( StandardLoggingHiddenParams, StandardLoggingMetadata, @@ -661,8 +661,6 @@ def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_ assert payload["model"] == "openai/gpt-4.1" assert payload["user"] == "test_user" - print(f"✅ Test passed! api_key preserved: {payload['api_key']}") - @pytest.mark.asyncio @patch("litellm.proxy.proxy_server.master_key", "sk-master-key") @@ -810,18 +808,6 @@ async def test_api_key_preserved_through_failure_hook_to_database(): assert payload.get("model") == "gpt-3.5-turbo" assert payload.get("user") == "test_user" - print("\n" + "=" * 80) - print("✅ CRITICAL E2E TEST PASSED") - print("=" * 80) - print(f"Token: {data['token']}") - print(f"Payload api_key: {payload_api_key}") - print(f"Match: {data['token'] == payload_api_key}") - print("=" * 80) - print("Production incident bug is FIXED and protected:") - print("- Failed requests preserve api_key through entire flow") - print("- Both SpendLogs AND DailyUserSpend will have correct api_key") - print("=" * 80 + "\n") - @patch("litellm.proxy.proxy_server.master_key", None) @patch("litellm.proxy.proxy_server.general_settings", {}) @@ -2073,3 +2059,255 @@ def test_sanitize_error_information_redacts_pydantic_assignment_form( assert sanitized is not None assert "leaked-via-pydantic-msg" not in sanitized["error_message"] assert REDACTED_BY_LITELM_STRING in sanitized["error_message"] + + +# ── _redact_logged_api_key unit tests ────────────────────────────────────── + + +def test_redact_logged_api_key_none_returns_none(): + assert _redact_logged_api_key(None) is None + + +def test_redact_logged_api_key_empty_string_returns_none(): + assert _redact_logged_api_key("") is None + + +def test_redact_logged_api_key_sk_key_is_hashed(): + raw = "sk-1234secret" + result = _redact_logged_api_key(raw) + assert result == hash_token(raw) + assert result is not None + assert not result.startswith("sk-") + assert len(result) == 64 + + +def test_redact_logged_api_key_bearer_sk_equals_sk_hash(): + raw = "sk-1234secret" + result_plain = _redact_logged_api_key(raw) + result_bearer = _redact_logged_api_key(f"Bearer {raw}") + assert result_bearer == result_plain + + +def test_redact_logged_api_key_bearer_case_insensitive(): + raw = "sk-1234secret" + result_lower = _redact_logged_api_key(f"bearer {raw}") + result_upper = _redact_logged_api_key(f"BEARER {raw}") + expected = hash_token(raw) + assert result_lower == expected + assert result_upper == expected + + +def test_redact_logged_api_key_non_sk_raw_key_is_hashed(): + raw = "anthropic-raw-key-xyz" + result = _redact_logged_api_key(raw) + assert result is not None + assert result != raw + assert len(result) == 64 + assert result == hash_token(raw) + + +def test_redact_logged_api_key_already_valid_sha256_passes_through_with_flag(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(already_hashed, already_hashed=True) + assert result == already_hashed + assert hash_token(already_hashed) != result # no double-hash + + +def test_redact_logged_api_key_sha256_without_flag_is_hashed(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(already_hashed) + assert result is not None + assert result != already_hashed + assert len(result) == 64 + assert result == hash_token(already_hashed) + + +def test_redact_logged_api_key_hashed_jwt_passes_through(): + jwt_hash = "hashed-jwt-" + "a" * 64 + result = _redact_logged_api_key(jwt_hash) + assert result == jwt_hash + + +def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed(): + short_jwt = "hashed-jwt-tooshort" + result = _redact_logged_api_key(short_jwt) + assert result is not None + assert result != short_jwt + assert len(result) == 64 + assert result == hash_token(short_jwt) + + +def test_redact_logged_api_key_bearer_only_returns_none(): + # "bearer " with nothing after stripping is equivalent to no key + assert _redact_logged_api_key("bearer ") is None + assert _redact_logged_api_key("Bearer ") is None + assert _redact_logged_api_key("BEARER ") is None + + +# ── _get_spend_logs_metadata key-hash invariant tests ───────────────────── + + +def test_get_spend_logs_metadata_sk_key_hashed(): + raw = "sk-1234secret" + meta = _get_spend_logs_metadata({"user_api_key": raw}) + assert meta["user_api_key"] == hash_token(raw) + assert meta["user_api_key"] is not None + result = meta["user_api_key"] + assert result is not None + assert not result.startswith("sk-") + assert len(result) == 64 + + +def test_get_spend_logs_metadata_bearer_sk_key_hashed_same_as_plain(): + raw = "sk-1234secret" + meta_plain = _get_spend_logs_metadata({"user_api_key": raw}) + meta_bearer = _get_spend_logs_metadata({"user_api_key": f"Bearer {raw}"}) + assert meta_bearer["user_api_key"] == meta_plain["user_api_key"] + + +def test_get_spend_logs_metadata_non_sk_raw_key_hashed(): + raw = "anthropic-raw-key-xyz" + meta = _get_spend_logs_metadata({"user_api_key": raw}) + result = meta["user_api_key"] + assert result is not None + assert result != raw + assert len(result) == 64 + + +def test_get_spend_logs_metadata_already_hashed_unchanged_with_provenance(): + already_hashed = hash_token("sk-some-key") + meta = _get_spend_logs_metadata( + {"user_api_key": already_hashed, "user_api_key_hash": already_hashed} + ) + assert meta["user_api_key"] == already_hashed + assert hash_token(already_hashed) != meta["user_api_key"] # no double-hash + + +def test_get_spend_logs_metadata_already_hashed_no_provenance_is_rehashed(): + already_hashed = hash_token("sk-some-key") + meta = _get_spend_logs_metadata({"user_api_key": already_hashed}) + assert meta["user_api_key"] != already_hashed + assert meta["user_api_key"] == hash_token(already_hashed) + + +def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match(): + already_hashed = hash_token("sk-some-key") + different_hash = hash_token("sk-other-key") + meta = _get_spend_logs_metadata( + {"user_api_key": already_hashed, "user_api_key_hash": different_hash} + ) + assert meta["user_api_key"] == hash_token(already_hashed) + + +def test_get_spend_logs_metadata_hashed_jwt_unchanged(): + jwt_hash = "hashed-jwt-" + "b" * 64 + meta = _get_spend_logs_metadata({"user_api_key": jwt_hash}) + assert meta["user_api_key"] == jwt_hash + + +def test_get_spend_logs_metadata_none_key_is_none(): + meta = _get_spend_logs_metadata({"user_api_key": None}) + assert meta["user_api_key"] is None + + +# ── get_logging_payload key-hash invariant tests ─────────────────────────── + + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_non_sk_raw_key_both_fields_hashed(): + raw = "anthropic-raw-key-xyz" + kwargs = { + "model": "openai/gpt-4.1", + "messages": [{"role": "user", "content": "Hello"}], + "call_type": "acompletion", + "litellm_params": { + "metadata": { + "user_api_key": raw, + "user_api_key_user_id": "test_user", + "user_api_key_team_id": "test_team", + } + }, + } + payload = get_logging_payload( + kwargs=kwargs, + response_obj=Exception("error"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["api_key"] != raw + assert len(payload["api_key"]) == 64 + + parsed_meta = json.loads(payload["metadata"]) + assert parsed_meta["user_api_key"] != raw + assert parsed_meta["user_api_key"] is not None + assert len(parsed_meta["user_api_key"]) == 64 + + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_empty_key_slp_none_is_empty_string_not_none_literal(): + kwargs = { + "model": "openai/gpt-4.1", + "messages": [{"role": "user", "content": "Hello"}], + "call_type": "acompletion", + "litellm_params": { + "metadata": { + "user_api_key_user_id": "test_user", + } + }, + } + payload = get_logging_payload( + kwargs=kwargs, + response_obj=Exception("error"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["api_key"] == "", ( + f"Expected empty string but got {payload['api_key']!r}; " + "dropping _redact_logged_api_key's 'or \"\"' guard would yield 'None' here" + ) + + +def test_get_spend_logs_metadata_sibling_fields_preserved(): + raw = "anthropic-raw-key-xyz" + meta = _get_spend_logs_metadata( + { + "user_api_key": raw, + "user_api_key_alias": "my-alias", + "user_api_key_team_id": "team-123", + } + ) + assert meta["user_api_key"] == hash_token(raw) + assert meta["user_api_key_alias"] == "my-alias" + assert meta["user_api_key_team_id"] == "team-123" + + +def test_redact_logged_api_key_partial_sha256_is_hashed(): + partial_hex = "a" * 63 + result = _redact_logged_api_key(partial_hex) + assert result is not None + assert result != partial_hex + assert len(result) == 64 + assert result == hash_token(partial_hex) + + +def test_redact_logged_api_key_bearer_already_hashed_passes_through_with_flag(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(f"Bearer {already_hashed}", already_hashed=True) + assert result == already_hashed + assert hash_token(already_hashed) != result + + +def test_redact_logged_api_key_bearer_sha256_without_flag_is_hashed(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(f"Bearer {already_hashed}") + assert result is not None + assert result != already_hashed + assert result == hash_token(already_hashed) From 9697748f92d1a02c4b2cfc2768c20c39242503d4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 22:28:43 -0700 Subject: [PATCH 2/5] test: gate the already-hashed pass-through on the provenance flag The spend-log helper no longer treats a 64-hex shape as proof a value was already hashed, so this case has to say where the hash came from. Reconciles the test that came in with #31799 against that change. --- .../proxy/spend_tracking/test_spend_tracking_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 706466d5033..e34d4f389e1 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2927,7 +2927,7 @@ class TestSpendLogKeyRedaction: def test_already_hashed_key_unchanged(self): hashed = "bcfe8173f5447f10be0e7fb37aaa8b97829d5c9e0498232152f9d123456789ab" - assert _redact_logged_api_key(hashed) == hashed + assert _redact_logged_api_key(hashed, already_hashed=True) == hashed def test_bearer_prefixed_non_sk_key_is_hashed(self): raw = "Bearer some-other-token-format" From fb417a556300fd6a983af66e28e0ee759d41a041 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 22:38:04 -0700 Subject: [PATCH 3/5] fix(spend-tracking): tie the already-hashed pass-through to provenance The hashed-jwt branch trusted the value's shape alone, so a caller-supplied key in that shape was stored unhashed. Both pass-throughs now require the value to match the auth-time user_api_key_hash, and the shape check is a full match. --- .../spend_tracking/spend_tracking_utils.py | 14 ++++++----- .../test_spend_tracking_utils.py | 25 +++++++++++++++++-- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 4592add1032..692200b856c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -64,7 +64,11 @@ def _is_master_key(api_key: str | None, _master_key: str | None) -> bool: return secrets.compare_digest(api_key, _master_key) -_HASHED_JWT_RE = re.compile(r"^hashed-jwt-[a-fA-F0-9]{64}$") +_HASHED_JWT_RE = re.compile(r"hashed-jwt-[a-fA-F0-9]{64}") + + +def _is_prehashed_key_shape(value: str) -> bool: + return is_valid_sha256_hash(value) or _HASHED_JWT_RE.fullmatch(value) is not None def _redact_logged_api_key(value: str | None, *, already_hashed: bool = False) -> str | None: @@ -73,9 +77,7 @@ def _redact_logged_api_key(value: str | None, *, already_hashed: bool = False) - stripped: Final = re.sub(r"(?i)^bearer ", "", value) if not stripped: return None - if already_hashed and is_valid_sha256_hash(stripped): - return stripped - if _HASHED_JWT_RE.match(stripped): + if already_hashed and _is_prehashed_key_shape(stripped): return stripped return hash_token(stripped) @@ -136,7 +138,7 @@ def _get_spend_logs_metadata( _raw_key: Final = clean_metadata.get("user_api_key") _trusted_hash: Final = metadata.get("user_api_key_hash") _already_hashed: Final = ( - isinstance(_trusted_hash, str) and is_valid_sha256_hash(_trusted_hash) and _trusted_hash == _raw_key + isinstance(_trusted_hash, str) and _is_prehashed_key_shape(_trusted_hash) and _trusted_hash == _raw_key ) clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_hashed=_already_hashed) clean_metadata["applied_guardrails"] = applied_guardrails @@ -296,7 +298,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs standard_logging_total_tokens = standard_logging_payload.get("total_tokens", 0) _trusted_hash = metadata.get("user_api_key_hash") _key_already_hashed = ( - isinstance(_trusted_hash, str) and is_valid_sha256_hash(_trusted_hash) and _trusted_hash == api_key + isinstance(_trusted_hash, str) and _is_prehashed_key_shape(_trusted_hash) and _trusted_hash == api_key ) api_key = _redact_logged_api_key(api_key, already_hashed=_key_already_hashed) or "" diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index e34d4f389e1..92e78e04512 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2653,10 +2653,24 @@ def test_redact_logged_api_key_long_opaque_token_is_hashed(): def test_redact_logged_api_key_hashed_jwt_passes_through(): jwt_hash = "hashed-jwt-" + "a" * 64 - result = _redact_logged_api_key(jwt_hash) + result = _redact_logged_api_key(jwt_hash, already_hashed=True) assert result == jwt_hash +def test_redact_logged_api_key_hashed_jwt_shape_without_provenance_is_hashed(): + lookalike = "hashed-jwt-" + "a" * 64 + result = _redact_logged_api_key(lookalike) + assert result == hash_token(lookalike) + assert result != lookalike + + +def test_redact_logged_api_key_hashed_jwt_trailing_newline_is_hashed(): + trailing = "hashed-jwt-" + "a" * 64 + "\n" + result = _redact_logged_api_key(trailing, already_hashed=True) + assert result == hash_token(trailing) + assert result != trailing + + def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed(): short_jwt = "hashed-jwt-tooshort" result = _redact_logged_api_key(short_jwt) @@ -2730,10 +2744,17 @@ def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match(): def test_get_spend_logs_metadata_hashed_jwt_unchanged(): jwt_hash = "hashed-jwt-" + "b" * 64 - meta = _get_spend_logs_metadata({"user_api_key": jwt_hash}) + meta = _get_spend_logs_metadata({"user_api_key": jwt_hash, "user_api_key_hash": jwt_hash}) assert meta["user_api_key"] == jwt_hash +def test_get_spend_logs_metadata_hashed_jwt_shape_without_provenance_is_hashed(): + lookalike = "hashed-jwt-" + "b" * 64 + meta = _get_spend_logs_metadata({"user_api_key": lookalike}) + assert meta["user_api_key"] == hash_token(lookalike) + assert meta["user_api_key"] != lookalike + + def test_get_spend_logs_metadata_none_key_is_none(): meta = _get_spend_logs_metadata({"user_api_key": None}) assert meta["user_api_key"] is None From c7b34da079e962d006e3b109739203703fa152f6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 23:19:17 -0700 Subject: [PATCH 4/5] test(proxy): keep a leaked llm_router out of the next test in the worker The proxy conftest already snapshots master_key and prisma_client around every test, because a value left behind on litellm.proxy.proxy_server poisons the rest of the xdist worker. llm_router has the same problem. The PTU rollup reads the running router out of sys.modules, so a router a sibling test left behind lands in its deployment scan and three test_ptu_flat_cost_rollup tests fail or pass depending on how xdist happens to split the shard. --- tests/test_litellm/proxy/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/conftest.py b/tests/test_litellm/proxy/conftest.py index 61752997f0f..65e12b7d777 100644 --- a/tests/test_litellm/proxy/conftest.py +++ b/tests/test_litellm/proxy/conftest.py @@ -18,6 +18,7 @@ from prisma.errors import ClientNotConnectedError _PROXY_MODULE_GLOBALS_TO_ISOLATE = ( "master_key", "prisma_client", + "llm_router", ) @@ -56,7 +57,10 @@ def pytest_runtest_setup(item): Without this, a leaked value (e.g. master_key set by a sibling test) flips the auth short-circuit in user_api_key_auth and causes unrelated - tests in the same xdist worker to return 401 instead of 200. + tests in the same xdist worker to return 401 instead of 200. A leaked + llm_router does the same to anything that reads the running router out + of sys.modules, such as the PTU rollup's deployment scan, which then + counts a sibling test's deployments as if the proxy owned them. This must be a hook pair, not an autouse fixture: an autouse fixture in the root conftest requests monkeypatch, so monkeypatch's undo stack From a50590f32454ee845775a04087d6cd42d249f37d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 23:41:55 -0700 Subject: [PATCH 5/5] fix(spend-tracking): keep the master key alias readable in spend logs Master-key auth stamps the stable alias litellm_proxy_master_key instead of the raw key, so spend logs carry a readable, non-secret identifier for those rows. The new redaction path only recognized sha256 and hashed-jwt shapes, so it hashed that alias and broke continuity with every master-key row written before this change. The alias joins the recognized non-secret values, still behind the same provenance gate, so a caller who sends the alias string as their own bearer token still gets it hashed. --- .../spend_tracking/spend_tracking_utils.py | 27 ++++---- .../test_spend_tracking_utils.py | 64 +++++++++++++++++-- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 692200b856c..cba1f9069d3 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -10,6 +10,7 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ( + LITELLM_PROXY_MASTER_KEY_ALIAS, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, REDACTED_BY_LITELM_STRING, @@ -67,17 +68,21 @@ def _is_master_key(api_key: str | None, _master_key: str | None) -> bool: _HASHED_JWT_RE = re.compile(r"hashed-jwt-[a-fA-F0-9]{64}") -def _is_prehashed_key_shape(value: str) -> bool: - return is_valid_sha256_hash(value) or _HASHED_JWT_RE.fullmatch(value) is not None +def _is_non_secret_key_value(value: str) -> bool: + return ( + value == LITELLM_PROXY_MASTER_KEY_ALIAS + or is_valid_sha256_hash(value) + or _HASHED_JWT_RE.fullmatch(value) is not None + ) -def _redact_logged_api_key(value: str | None, *, already_hashed: bool = False) -> str | None: +def _redact_logged_api_key(value: str | None, *, already_redacted: bool = False) -> str | None: if not isinstance(value, str) or not value: return None stripped: Final = re.sub(r"(?i)^bearer ", "", value) if not stripped: return None - if already_hashed and _is_prehashed_key_shape(stripped): + if already_redacted and _is_non_secret_key_value(stripped): return stripped return hash_token(stripped) @@ -137,10 +142,10 @@ def _get_spend_logs_metadata( clean_metadata: Final = SpendLogsMetadata(**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__}) _raw_key: Final = clean_metadata.get("user_api_key") _trusted_hash: Final = metadata.get("user_api_key_hash") - _already_hashed: Final = ( - isinstance(_trusted_hash, str) and _is_prehashed_key_shape(_trusted_hash) and _trusted_hash == _raw_key + _already_redacted: Final = ( + isinstance(_trusted_hash, str) and _is_non_secret_key_value(_trusted_hash) and _trusted_hash == _raw_key ) - clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_hashed=_already_hashed) + clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_redacted=_already_redacted) clean_metadata["applied_guardrails"] = applied_guardrails clean_metadata["batch_models"] = batch_models clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata @@ -297,10 +302,10 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs standard_logging_completion_tokens = standard_logging_payload.get("completion_tokens", 0) standard_logging_total_tokens = standard_logging_payload.get("total_tokens", 0) _trusted_hash = metadata.get("user_api_key_hash") - _key_already_hashed = ( - isinstance(_trusted_hash, str) and _is_prehashed_key_shape(_trusted_hash) and _trusted_hash == api_key + _key_already_redacted = ( + isinstance(_trusted_hash, str) and _is_non_secret_key_value(_trusted_hash) and _trusted_hash == api_key ) - api_key = _redact_logged_api_key(api_key, already_hashed=_key_already_hashed) or "" + api_key = _redact_logged_api_key(api_key, already_redacted=_key_already_redacted) or "" if ( standard_logging_payload is not None @@ -308,7 +313,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs api_key = ( api_key or _redact_logged_api_key( - standard_logging_payload["metadata"].get("user_api_key_hash"), already_hashed=True + standard_logging_payload["metadata"].get("user_api_key_hash"), already_redacted=True ) or "" ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 92e78e04512..f2dd66ee677 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2625,7 +2625,7 @@ def test_redact_logged_api_key_non_sk_raw_key_is_hashed(): def test_redact_logged_api_key_already_valid_sha256_passes_through_with_flag(): already_hashed = hash_token("sk-some-key") assert len(already_hashed) == 64 - result = _redact_logged_api_key(already_hashed, already_hashed=True) + result = _redact_logged_api_key(already_hashed, already_redacted=True) assert result == already_hashed assert hash_token(already_hashed) != result # no double-hash @@ -2653,7 +2653,7 @@ def test_redact_logged_api_key_long_opaque_token_is_hashed(): def test_redact_logged_api_key_hashed_jwt_passes_through(): jwt_hash = "hashed-jwt-" + "a" * 64 - result = _redact_logged_api_key(jwt_hash, already_hashed=True) + result = _redact_logged_api_key(jwt_hash, already_redacted=True) assert result == jwt_hash @@ -2666,7 +2666,7 @@ def test_redact_logged_api_key_hashed_jwt_shape_without_provenance_is_hashed(): def test_redact_logged_api_key_hashed_jwt_trailing_newline_is_hashed(): trailing = "hashed-jwt-" + "a" * 64 + "\n" - result = _redact_logged_api_key(trailing, already_hashed=True) + result = _redact_logged_api_key(trailing, already_redacted=True) assert result == hash_token(trailing) assert result != trailing @@ -2680,6 +2680,33 @@ def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed(): assert result == hash_token(short_jwt) +def test_redact_logged_api_key_master_key_alias_passes_through(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS, already_redacted=True) + assert result == LITELLM_PROXY_MASTER_KEY_ALIAS + + +def test_redact_logged_api_key_master_key_alias_without_provenance_is_hashed(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS) + assert result == hash_token(LITELLM_PROXY_MASTER_KEY_ALIAS) + assert result != LITELLM_PROXY_MASTER_KEY_ALIAS + + +def test_get_spend_logs_metadata_keeps_master_key_alias_readable(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + meta = _get_spend_logs_metadata( + { + "user_api_key": LITELLM_PROXY_MASTER_KEY_ALIAS, + "user_api_key_hash": LITELLM_PROXY_MASTER_KEY_ALIAS, + } + ) + assert meta["user_api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS + + def test_redact_logged_api_key_bearer_only_returns_none(): # "bearer " with nothing after stripping is equivalent to no key assert _redact_logged_api_key("bearer ") is None @@ -2948,7 +2975,7 @@ class TestSpendLogKeyRedaction: def test_already_hashed_key_unchanged(self): hashed = "bcfe8173f5447f10be0e7fb37aaa8b97829d5c9e0498232152f9d123456789ab" - assert _redact_logged_api_key(hashed, already_hashed=True) == hashed + assert _redact_logged_api_key(hashed, already_redacted=True) == hashed def test_bearer_prefixed_non_sk_key_is_hashed(self): raw = "Bearer some-other-token-format" @@ -2995,6 +3022,33 @@ def test_get_logging_payload_non_sk_raw_key_both_fields_hashed(): assert len(parsed_meta["user_api_key"]) == 64 +def test_get_logging_payload_keeps_master_key_alias_readable(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + kwargs = { + "model": "openai/gpt-4.1", + "messages": [{"role": "user", "content": "Hello"}], + "call_type": "acompletion", + "litellm_params": { + "metadata": { + "user_api_key": LITELLM_PROXY_MASTER_KEY_ALIAS, + "user_api_key_hash": LITELLM_PROXY_MASTER_KEY_ALIAS, + "user_api_key_user_id": "test_user", + } + }, + } + payload = get_logging_payload( + kwargs=kwargs, + response_obj=Exception("error"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS + parsed_meta = json.loads(payload["metadata"]) + assert parsed_meta["user_api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS + + @patch("litellm.proxy.proxy_server.master_key", None) @patch("litellm.proxy.proxy_server.general_settings", {}) def test_get_logging_payload_hashes_bearer_prefixed_api_key(): @@ -3503,7 +3557,7 @@ def test_redact_logged_api_key_partial_sha256_is_hashed(): def test_redact_logged_api_key_bearer_already_hashed_passes_through_with_flag(): already_hashed = hash_token("sk-some-key") assert len(already_hashed) == 64 - result = _redact_logged_api_key(f"Bearer {already_hashed}", already_hashed=True) + result = _redact_logged_api_key(f"Bearer {already_hashed}", already_redacted=True) assert result == already_hashed assert hash_token(already_hashed) != result