diff --git a/litellm/_logging.py b/litellm/_logging.py index 9435562f890..14cda772234 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -6,6 +6,7 @@ import sys from datetime import datetime from logging import Formatter from typing import Any, Final, TextIO +from urllib.parse import unquote import litellm from litellm.constants import ( @@ -146,6 +147,72 @@ class SecretRedactionFilter(logging.Filter): _secret_filter: Final = SecretRedactionFilter() +_MAX_SCRUBBED_ACCESS_ARG: Final = 512 + +_REDACTION_PLACEHOLDER: Final = "REDACTED" + + +def _hides_a_credential(value: str) -> bool: + """Whether *value* only looks clean until it is percent-decoded.""" + decoded: Final = unquote(value) + return _redact_string(decoded) != decoded + + +def _drop_encoded_credential(scrubbed: str) -> str: + """Drop the part of a request target that only decoding shows to be a secret. + + The request parser decodes query names and values, so `?k%65y=sk%2D...` is a + working credential that the patterns, which match literal text, do not see. + The decoded text is never logged back: it can carry a newline, and forging + log lines is not a trade worth making for a readable request target. + """ + path, separator, _query = scrubbed.partition("?") + if _hides_a_credential(path): + return _REDACTION_PLACEHOLDER + if separator and _hides_a_credential(scrubbed): + return f"{path}?{_REDACTION_PLACEHOLDER}" + return scrubbed + + +def _scrub_access_arg(value: str) -> str: + """Redact one access-log positional arg, bounding the scanned length. + + The request target is the only input to the secret regex an unauthenticated + caller controls end to end, so it is cut back to a whole query parameter + before it is scanned; a half-parameter would be too short to match its + pattern and would then be logged raw. + """ + if len(value) <= _MAX_SCRUBBED_ACCESS_ARG: + return _drop_encoded_credential(_redact_string(value)) + head: Final = value[:_MAX_SCRUBBED_ACCESS_ARG] + kept: Final = head[: max(head.rfind("?"), head.rfind("&"))] if "?" in head else head + scrubbed: Final = _drop_encoded_credential(_redact_string(kept)) + return f"{scrubbed}... ({len(value) - len(kept)} more chars truncated) ..." + + +class AccessLogRedactionFilter(logging.Filter): + """Scrubs known secret/credential patterns from HTTP access-log records. + + uvicorn's AccessFormatter unpacks ``record.args`` as a five-element tuple at + emit time, so SecretRedactionFilter cannot be reused here: it collapses the + record into ``record.msg`` and clears the args, and the formatter then raises. + """ + + def filter(self, record: logging.LogRecord) -> bool: + if not _ENABLE_SECRET_REDACTION: + return True + if isinstance(record.args, tuple) and record.args: + record.args = tuple( # rebind-ok: a Filter scrubs records in place + _scrub_access_arg(arg) if isinstance(arg, str) else arg for arg in record.args + ) + return True + # No positional args means everything is in msg, where collapsing is correct. + return _secret_filter.filter(record) + + +_access_log_filter: Final = AccessLogRedactionFilter() + + def _get_max_string_length_stdout_log() -> int: """Read the limit per record so a value loaded later via proxy config environment_variables is honored.""" @@ -553,6 +620,14 @@ _REDACTED_THIRD_PARTY_LOGGERS: Final[tuple[str, ...]] = ( "uvicorn.error", ) +# Access loggers, which emit the full request target, so a credential passed as a +# query parameter (e.g. `/key/info?key=`) lands on stdout verbatim. uvicorn.access +# covers uvicorn.run, --run_gunicorn (its worker_class is UvicornWorker, so the +# access line is still uvicorn's) and an embedding host app. --run_hypercorn and +# --run_granian log through their own loggers in their own record shapes, and +# both ship with access logging off. +_REDACTED_ACCESS_LOGGERS: Final[tuple[str, ...]] = ("uvicorn.access",) + def _redact_third_party_loggers() -> None: """Extend secret redaction to records litellm does not emit directly. @@ -575,6 +650,8 @@ def _redact_third_party_loggers() -> None: """ for name in _REDACTED_THIRD_PARTY_LOGGERS: logging.getLogger(name).addFilter(_secret_filter) + for name in _REDACTED_ACCESS_LOGGERS: + logging.getLogger(name).addFilter(_access_log_filter) # Call the suppression function diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py index 5d5bd547d22..e93ab155786 100644 --- a/litellm/litellm_core_utils/secret_redaction.py +++ b/litellm/litellm_core_utils/secret_redaction.py @@ -30,6 +30,11 @@ def _build_secret_patterns() -> "re.Pattern[str]": r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}", # OpenAI / Anthropic sk- prefixed keys rf"sk-[A-Za-z0-9\-_]{{{MINIMUM_CUSTOM_KEY_LENGTH - len('sk-')},}}", + # Credentials passed as URL query params. Terminated by "&" like the key= + # and sig= patterns below, so the rest of the request line survives in an + # access log. Must precede the generic patterns to win at the same position. + r"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))" + r"=[^\s&'\"]+", # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr) r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}", # x-api-key / api-key header values (handles 'key': 'value' dict repr) @@ -45,8 +50,10 @@ def _build_secret_patterns() -> "re.Pattern[str]": # Word boundary prevents O(n^2) backtracking on long word-char runs. r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)" r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", - # Database connection string credentials (scheme://user:pass@host) - r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)", + # Database connection string credentials (scheme://user:pass@host). + # The user half stops at the ":" separator and both halves are length-capped, + # so a long attacker-supplied URL cannot backtrack quadratically. + r"(?<=://)[^\s'\":]{0,4096}:[^\s'\"]{1,4096}(?=@)", # Databricks personal access tokens r"dapi[0-9a-f]{32}", # Module-level provider keys logged as litellm._key= @@ -67,8 +74,10 @@ def _build_secret_patterns() -> "re.Pattern[str]": r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""", # Raw JWTs (without Bearer prefix) r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*", - # Azure SAS tokens in URLs - r"[?&]sig=[A-Za-z0-9%+/=]+", + # Azure SAS tokens in URLs. The delimiter is a lookbehind, like the + # `key=` pattern above, so the `?` or `&` survives and the redacted URL + # stays well formed (this string is often a request line in a log). + r"(?<=[?&])sig=[A-Za-z0-9%+/=]+", # Full JSON service-account blobs (single-line and multi-line) r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', ] diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 5703c6cd5e8..0db4cbc3bf2 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -4705,6 +4705,13 @@ async def is_valid_fallback_model( return True +# The shape abbreviate_api_key writes into LiteLLM_VerificationToken.key_name. The +# last four characters are only barred from being whitespace or a control code, +# because a custom key's can be anything else, punctuation and non-ASCII included; +# a real key is at least MINIMUM_CUSTOM_KEY_LENGTH long, so it never fullmatches. +_MASKED_KEY_NAME_RE: Final = re.compile(r"sk-\.\.\.(?:[^\s\x00-\x1f\x7f-\x9f]{4})?") + + def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool: """ Throttle an over-budget key instead of blocking it, when the key opted in @@ -4785,10 +4792,15 @@ async def _virtual_key_max_budget_check( if math.isfinite(valid_token.max_budget) and spend >= valid_token.max_budget: if _apply_budget_exceeded_throttle(valid_token): return - # name the key in the error so operators don't have to reverse-map - # spend back to a key; key_name is the masked form (last 4 chars) + # This message is returned to the caller, and key_name has no enforced + # shape (a direct DB write bypasses abbreviate_api_key), so echo it only + # when it still looks masked and fall back to the alias otherwise. key_label: Final = valid_token.key_alias or "key" - key_descriptor: Final = f"{key_label} ({valid_token.key_name})" if valid_token.key_name else key_label + key_descriptor: Final = ( + f"{key_label} ({valid_token.key_name})" + if valid_token.key_name and _MASKED_KEY_NAME_RE.fullmatch(valid_token.key_name) + else key_label + ) raise litellm.BudgetExceededError( current_cost=spend, max_budget=valid_token.max_budget, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c3403cf477c..d7d20d168b5 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3785,15 +3785,22 @@ async def info_key_fn_v2( @router.get("/key/info", tags=["key management"], dependencies=[Depends(user_api_key_auth)]) @management_endpoint_wrapper async def info_key_fn( - key: str | None = fastapi.Query(default=None, description="Key in the request parameters"), + key: str | None = fastapi.Query( + default=None, + description=( + "Key to look up. Pass the key's sha256 hash so the raw key stays out of URLs and access " + "logs. Example key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa'" + ), + ), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ Retrieve information about a key. Parameters: - - key: str | None (query parameter) - The key to look up. Accepts the plaintext key or its hash. - Defaults to the key in the Authorization header. + - key: str | None (query parameter) - The key to look up. Accepts the plaintext key or its hash; + prefer the hash, since a query parameter is recorded verbatim by any HTTP access log in front + of the proxy. Defaults to the key in the Authorization header. Returns: - key: str - The key that was looked up, echoed back as it was passed in @@ -3825,7 +3832,7 @@ async def info_key_fn( Example Curl: ``` - curl -X GET "http://0.0.0.0:4000/key/info?key=sk-test-example-key-123" \ + curl -X GET "http://0.0.0.0:4000/key/info?key=d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa" \ -H "Authorization: Bearer sk-1234" ``` diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 41c65b1d5c5..c48750cea72 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1204,7 +1204,10 @@ async def get_global_spend_report( ), api_key: str | None = fastapi.Query( default=None, - description="View spend for a specific api_key. Example api_key='sk-1234", + description=( + "View spend for a specific api_key. Pass the key's sha256 hash so the raw key stays " + "out of URLs and access logs. Example api_key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa'" + ), ), internal_user_id: str | None = fastapi.Query( default=None, @@ -1685,7 +1688,11 @@ async def get_key_spend_report( api_key: Annotated[ str | None, fastapi.Query( - description="View spend for a specific api_key. Proxy admin only; other callers are scoped to their own key." + description=( + "View spend for a specific api_key. Proxy admin only; other callers are scoped to their " + "own key. Pass the key's sha256 hash so the raw key stays out of URLs and access logs. " + "Example api_key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa'" + ) ), ] = None, ) -> Sequence[Mapping[str, object]]: @@ -2945,7 +2952,7 @@ async def view_spend_logs( Example Request for specific api_key ``` - curl -X GET "http://0.0.0.0:8000/spend/logs?api_key=sk-test-example-key-123" \ + curl -X GET "http://0.0.0.0:8000/spend/logs?api_key=d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa" \ -H "Authorization: Bearer sk-1234" ``` diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 3dea89ed67b..ffe3e0fee12 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -7448,3 +7448,64 @@ async def test_delete_cache_key_object_is_best_effort_when_the_cache_backend_fai healthy_cache.delete_cache.assert_called_once_with(key=hashed_token) healthy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache.assert_awaited_once_with(key=hashed_token) assert caplog.records == [], "a healthy eviction must stay silent, and must still reach both caches" + + +# --------------------------------------------------------------------------- +# Budget-exceeded error text must not carry a raw virtual key (LIT-5909) +# --------------------------------------------------------------------------- + + +class _BudgetAlertRecorder: + async def budget_alerts(self, type, user_info): + return None + + +async def _run_key_budget_check(key_name: str) -> str: + """Drive the over-budget key path and return the raised message.""" + valid_token = UserAPIKeyAuth( + token="hashed-token", + key_name=key_name, + key_alias="prod-key", + spend=10.0, + max_budget=1.0, + ) + with pytest.raises(litellm.BudgetExceededError, match="Budget has been exceeded") as exc_info: + await _virtual_key_max_budget_check( + valid_token=valid_token, + proxy_logging_obj=_BudgetAlertRecorder(), + ) + await asyncio.sleep(0) + return exc_info.value.message + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_name", + [ + "sk-mx5ous1o9Iezz5fj3pkLuA", + "my-company-key-2026", + "sk-...5LuA-but-longer", + # /key/generate takes a custom key ending in an escape sequence, and this + # message reaches a terminal and a log viewer + "sk-...\x1b[2J", + "sk-...a\x9bm", + ], +) +async def test_key_budget_error_does_not_carry_a_raw_key_name(key_name): + """key_name is written masked, but the column has no enforced shape (a direct DB + write bypasses abbreviate_api_key) and this message is returned to the caller.""" + message = await _run_key_budget_check(key_name) + assert key_name not in message + assert "Key=prod-key Current cost" in message + + +@pytest.mark.asyncio +@pytest.mark.parametrize("key_name", ["sk-...5LuA", "sk-...", "sk-...ke.!", "sk-...café"]) +async def test_key_budget_error_keeps_the_masked_key_name(key_name): + """The masked form is the whole point of naming the key, so it must survive. + + abbreviate_api_key takes the last four characters of the key verbatim, and a + custom key may end in punctuation or a non-ASCII character, so those masked + names are just as valid as the alphanumeric ones.""" + message = await _run_key_budget_check(key_name) + assert f"Key=prod-key ({key_name}) Current cost" in message diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py index 087a1c8b3ad..bf3757e6886 100644 --- a/tests/test_litellm/test_logging.py +++ b/tests/test_litellm/test_logging.py @@ -1,26 +1,30 @@ import ast import asyncio import json +import logging import re import sys +import time +from io import StringIO from pathlib import Path from typing import List import pytest -import logging - import litellm from litellm._logging import ( _COLOR_LOG_FORMAT, + _MAX_SCRUBBED_ACCESS_ARG, _PLAIN_LOG_FORMAT, ALL_LOGGERS, + AccessLogRedactionFilter, CorrelationContextFilter, CorrelationPlainFormatter, JsonFormatter, LevelRoutingStreamHandler, SecretRedactionFilter, StdoutLogTruncationFilter, + _get_uvicorn_json_log_config, _initialize_loggers_with_handler, _parse_json_logs_env, _plain_log_format, @@ -968,3 +972,209 @@ def test_plain_log_format_survives_none_streams(): """sys.stdout/sys.stderr can be None in embedded interpreters; import must not crash.""" assert _plain_log_format(None, None) == _PLAIN_LOG_FORMAT assert _plain_log_format(_FakeStream(True), None) == _PLAIN_LOG_FORMAT + + +# --------------------------------------------------------------------------- +# Access-log redaction (LIT-5909) +# --------------------------------------------------------------------------- + +_LEAKED_KEY = "sk-mx5ous1o9Iezz5fj3pkLuA" + + +def _access_record(full_path: str) -> logging.LogRecord: + """A record shaped exactly like the one uvicorn.access emits per request.""" + return logging.LogRecord( + name="uvicorn.access", + level=logging.INFO, + pathname="", + lineno=0, + msg='%s - "%s %s HTTP/%s" %d', + args=("127.0.0.1:1", "GET", full_path, "1.1", 200), + exc_info=None, + ) + + +@pytest.mark.parametrize( + "full_path", + [ + f"/key/info?key={_LEAKED_KEY}", + f"/global/spend/report?api_key={_LEAKED_KEY}&start_date=2026-08-01", + f"/key/spend/report?api_key={_LEAKED_KEY}", + f"/spend/logs?api_key={_LEAKED_KEY}", + f"/user/daily/activity?api_key={_LEAKED_KEY}", + f"/gemini/v1beta/models/gemini-2.0-flash:generateContent?key={_LEAKED_KEY}", + ], +) +def test_access_log_filter_redacts_a_credential_query_parameter(full_path): + record = _access_record(full_path) + assert AccessLogRedactionFilter().filter(record) is True + assert _LEAKED_KEY not in record.getMessage() + assert "REDACTED" in record.getMessage() + + +def test_access_log_filter_keeps_the_record_formattable_by_uvicorn(): + """uvicorn's AccessFormatter unpacks record.args, so the filter must scrub the + args in place rather than collapse them the way SecretRedactionFilter does.""" + from uvicorn.logging import AccessFormatter + + record = _access_record(f"/key/info?key={_LEAKED_KEY}") + AccessLogRedactionFilter().filter(record) + assert isinstance(record.args, tuple) + assert len(record.args) == 5 + + formatted = AccessFormatter('%(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False).format(record) + assert _LEAKED_KEY not in formatted + assert "GET" in formatted + assert "200 OK" in formatted + + +@pytest.mark.parametrize( + "full_path, want", + [ + # The delimiter must survive so the logged request line stays well formed. + (f"/key/info?key={_LEAKED_KEY}&page=2", "/key/info?REDACTED&page=2"), + ("/download?sig=AbCd1234%2Fxy&page=2", "/download?REDACTED&page=2"), + ( + f"/global/spend/report?api_key={_LEAKED_KEY}&start_date=2026-01-01", + "/global/spend/report?REDACTED&start_date=2026-01-01", + ), + ("/sso/callback?client_secret=abcdefgh12345&state=xyz", "/sso/callback?REDACTED&state=xyz"), + (f"/v1/models?token={_LEAKED_KEY}&page=2", "/v1/models?REDACTED&page=2"), + ], +) +def test_access_log_filter_keeps_the_query_delimiter(full_path, want): + record = _access_record(full_path) + AccessLogRedactionFilter().filter(record) + assert record.args[2] == want + + +@pytest.mark.parametrize( + "full_path, want", + [ + # Both the param name and the value are encoded, so neither is literal text + # the patterns can see, yet the request parser decodes it into a working key. + (f"/key/info?k%65y=sk%2D{_LEAKED_KEY[3:]}", "/key/info?REDACTED"), + (f"/key/info?k%65y=sk%2D{_LEAKED_KEY[3:]}&page=2", "/key/info?REDACTED"), + (f"/v1/models/sk%2D{_LEAKED_KEY[3:]}", "REDACTED"), + # A decoded credential must never be echoed back: it can carry a newline and + # forge a following log line. + (f"/v1/models?k%65y=sk%2D{_LEAKED_KEY[3:]}%0AINFO:%20forged", "/v1/models?REDACTED"), + ], +) +def test_access_log_filter_redacts_a_percent_encoded_credential(full_path, want): + record = _access_record(full_path) + AccessLogRedactionFilter().filter(record) + assert record.args[2] == want + + +@pytest.mark.parametrize( + "full_path", + [ + "/v1/models?filter=gpt%2D4o&page=2", + "/gemini/v1beta/models/gemini-2.0-flash%3AgenerateContent", + ], +) +def test_access_log_filter_leaves_harmless_percent_encoding_alone(full_path): + """Decoding is a detector, not a rewrite, so a request line with no credential + in it survives encoded exactly as the client sent it.""" + record = _access_record(full_path) + AccessLogRedactionFilter().filter(record) + assert record.args[2] == full_path + + +def test_access_log_filter_caps_how_much_of_a_request_target_it_scans(): + """The request target is the only input to the secret regex an unauthenticated + caller controls end to end, so it is bounded before it is scanned, and the + dropped tail must not reach the log either.""" + record = _access_record("/v1/models?u=" + "a://" * 8192 + f"&key={_LEAKED_KEY}") + + started = time.perf_counter() + AccessLogRedactionFilter().filter(record) + elapsed = time.perf_counter() - started + + scrubbed = record.args[2] + assert _LEAKED_KEY not in scrubbed + assert len(scrubbed) < 1024 + assert elapsed < 1.0, f"scrubbing one access line took {elapsed:.2f}s" + + +@pytest.mark.parametrize("chars_before_the_cut", range(1, 12)) +def test_access_log_filter_never_logs_a_half_scanned_credential(chars_before_the_cut): + """Cutting mid-value would leave a prefix too short for the key= pattern to match, + and that prefix would then be logged raw, so the cut lands on a param boundary.""" + prefix = "/v1/models?u=" + padding = _MAX_SCRUBBED_ACCESS_ARG - len(prefix) - len("&key=") - chars_before_the_cut + record = _access_record(f"{prefix}{'a' * padding}&key={_LEAKED_KEY}") + + AccessLogRedactionFilter().filter(record) + + assert f"key={_LEAKED_KEY[:chars_before_the_cut]}" not in record.args[2] + + +def test_access_log_filter_leaves_a_credential_free_request_line_intact(): + record = _access_record("/v1/chat/completions") + AccessLogRedactionFilter().filter(record) + assert record.getMessage() == '127.0.0.1:1 - "GET /v1/chat/completions HTTP/1.1" 200' + + +def test_access_log_filter_redacts_a_record_that_carries_no_positional_args(): + record = logging.LogRecord( + name="uvicorn.access", + level=logging.INFO, + pathname="", + lineno=0, + msg=f'127.0.0.1:1 - "GET /key/info?key={_LEAKED_KEY} HTTP/1.1" 200', + args=None, + exc_info=None, + ) + assert AccessLogRedactionFilter().filter(record) is True + assert _LEAKED_KEY not in record.getMessage() + + +def _emit_access_line(full_path: str) -> str: + """Hand one real record to uvicorn.access and return what a handler wrote out.""" + from uvicorn.logging import AccessFormatter + + logger = logging.getLogger("uvicorn.access") + stream = StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(AccessFormatter('%(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False)) + saved_level, saved_propagate = logger.level, logger.propagate + logger.addHandler(handler) + logger.setLevel(logging.INFO) + logger.propagate = False + try: + logger.handle(_access_record(full_path)) + finally: + logger.removeHandler(handler) + logger.setLevel(saved_level) + logger.propagate = saved_propagate + return stream.getvalue() + + +def test_uvicorn_access_logger_redacts_a_credential_it_is_handed(): + """Registration happens at litellm import; without it the filter never runs.""" + emitted = _emit_access_line(f"/key/info?key={_LEAKED_KEY}") + + assert _LEAKED_KEY not in emitted + assert "REDACTED" in emitted + + +def test_access_redaction_survives_the_uvicorn_json_log_config(): + """litellm hands uvicorn a dictConfig when json_logs is on. dictConfig clears a + logger's handlers but not its filters, so redaction has to still be attached.""" + import logging.config + + names = ("uvicorn", "uvicorn.error", "uvicorn.access") + saved = tuple((logging.getLogger(n), logging.getLogger(n).handlers[:], logging.getLogger(n).level) for n in names) + try: + logging.config.dictConfig(_get_uvicorn_json_log_config()) + emitted = _emit_access_line(f"/key/info?key={_LEAKED_KEY}") + + assert _LEAKED_KEY not in emitted + assert "REDACTED" in emitted + finally: + for lg, handlers, level in saved: + lg.handlers[:] = handlers + lg.setLevel(level) + lg.propagate = True diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 7d50694c805..303efcab9c7 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -1,8 +1,10 @@ import logging import logging.config import sys +import time from collections.abc import Callable from io import StringIO +from typing import Final from unittest.mock import patch import pytest @@ -67,6 +69,50 @@ def test_redact_string_catches_secret_patterns(): assert redact_string(normal) == normal +@pytest.mark.parametrize( + "connection_string", + [ + "postgres://admin:pass3cret@db.example.com:5432/mydb", + "redis://:pass3cret@cache.example.com:6379", + "postgres://admin:pass/s3cret@db.example.com:5432/mydb", + "amqp://admin:pass:s3cret@rabbit:5672", + "https://ad@min:pass3cret@host", + # An unencoded "@" inside the password, with a ":" after it + "postgresql://admin:p@ss3cret:2026@db.example.com:5432/mydb", + "amqp://guest:gu@st3cret:1@rabbit:5672/", + # An AWS RDS IAM auth token is a presigned query string used as the + # password, so the userinfo runs to several hundred characters. + "postgresql://litellm:host%3A5432%2F%3FAction%3Dconnect%26X-Amz-Signature%3D" + + "f" * 540 + + "s3cret@db.host:5432/litellm", + ], +) +def test_redact_string_still_catches_connection_string_credentials(connection_string): + """The bounded userinfo pattern must keep matching real connection strings.""" + assert "s3cret" not in redact_string(connection_string) + + +def _redaction_cost(url_bytes: int) -> float: + url: Final = "/x?u=" + "a://" * (url_bytes // 4) + + def once() -> float: + started = time.perf_counter() + redact_string(url) + return time.perf_counter() - started + + return min(once() for _ in range(3)) + + +def test_redact_string_stays_sub_quadratic_on_a_long_adversarial_url(): + """Access-log redaction runs on attacker-controlled request lines, so quadrupling + a URL of scheme separators must not multiply the cost by sixteen. Comparing two + sizes rather than asserting a wall-clock ceiling keeps this honest on a slow box: + the unbounded pattern this replaced cost 5s at 4 KB and 314s at 16 KB.""" + growth: Final = _redaction_cost(16 * 1024) / _redaction_cost(4 * 1024) + + assert growth < 11.0, f"cost grew {growth:.1f}x for 4x the URL length" + + def test_redact_string_catches_minimum_length_virtual_key(): """Regression test for LIT-4355: keys at the enforced 16-char minimum (MINIMUM_CUSTOM_KEY_LENGTH) must be treated as key-shaped by the scrubber.""" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9f5ad97f71d..522a2de574b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7712,8 +7712,9 @@ export interface paths { * @description Retrieve information about a key. * * Parameters: - * - key: str | None (query parameter) - The key to look up. Accepts the plaintext key or its hash. - * Defaults to the key in the Authorization header. + * - key: str | None (query parameter) - The key to look up. Accepts the plaintext key or its hash; + * prefer the hash, since a query parameter is recorded verbatim by any HTTP access log in front + * of the proxy. Defaults to the key in the Authorization header. * * Returns: * - key: str - The key that was looked up, echoed back as it was passed in @@ -7745,7 +7746,7 @@ export interface paths { * * Example Curl: * ``` - * curl -X GET "http://0.0.0.0:4000/key/info?key=sk-test-example-key-123" -H "Authorization: Bearer sk-1234" + * curl -X GET "http://0.0.0.0:4000/key/info?key=d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa" -H "Authorization: Bearer sk-1234" * ``` * * Example Curl - if no key is passed, it will use the Key Passed in Authorization Header @@ -14041,7 +14042,7 @@ export interface paths { * * Example Request for specific api_key * ``` - * curl -X GET "http://0.0.0.0:8000/spend/logs?api_key=sk-test-example-key-123" -H "Authorization: Bearer sk-1234" + * curl -X GET "http://0.0.0.0:8000/spend/logs?api_key=d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa" -H "Authorization: Bearer sk-1234" * ``` * * Example Request for specific user_id @@ -47485,7 +47486,7 @@ export interface operations { end_date?: string | null; /** @description Group spend by internal team or customer or api_key */ group_by?: ("team" | "customer" | "api_key") | null; - /** @description View spend for a specific api_key. Example api_key='sk-1234 */ + /** @description View spend for a specific api_key. Pass the key's sha256 hash so the raw key stays out of URLs and access logs. Example api_key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa' */ api_key?: string | null; /** @description View spend for a specific internal_user_id. Example internal_user_id='1234 */ internal_user_id?: string | null; @@ -49358,7 +49359,7 @@ export interface operations { info_key_fn_key_info_get: { parameters: { query?: { - /** @description Key in the request parameters */ + /** @description Key to look up. Pass the key's sha256 hash so the raw key stays out of URLs and access logs. Example key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa' */ key?: string | null; }; header?: never; @@ -49536,7 +49537,7 @@ export interface operations { start_date?: string | null; /** @description Time till which to view spend (YYYY-MM-DD) */ end_date?: string | null; - /** @description View spend for a specific api_key. Proxy admin only; other callers are scoped to their own key. */ + /** @description View spend for a specific api_key. Proxy admin only; other callers are scoped to their own key. Pass the key's sha256 hash so the raw key stays out of URLs and access logs. Example api_key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa' */ api_key?: string | null; }; header?: never;