diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 41f13817d75..7a4c9688781 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -431,6 +431,10 @@ _INPUT_PRICING_KEY_PREFIXES: Final = ( _OUTPUT_PRICING_KEY_PREFIXES: Final = ("output_cost_per_token",) _BATCH_PRICING_KEY_SUFFIX: Final = "_batches" +# Proxy-internal callbacks that must run even on `no-log` requests: the proxy still has to do +# its own spend accounting. The `_PROXY_` handler prefix marks most of them; the spend writer +# `_ProxyDBLogger` predates that prefix, so it is listed explicitly (exact names, no substring). +_PROXY_INTERNAL_CALLBACK_CLASS_NAMES: Final = frozenset({"_ProxyDBLogger"}) _NO_CARRIED_RATES: Final[Mapping[str, object]] = MappingProxyType({}) @@ -2647,9 +2651,15 @@ class Logging(LiteLLMLoggingBaseClass): return True if litellm_params.get("no-log", False) is True: - # proxy cost tracking cal backs should run - - if not (isinstance(callback, CustomLogger) and "_PROXY_" in callback.__class__.__name__): + # Proxy-internal callbacks must still run on `no-log` requests: the proxy's own + # spend accounting has to record the request. They use the internal `_PROXY_` + # handler prefix or -- for the spend writer, which predates that prefix -- an + # explicit entry in `_PROXY_INTERNAL_CALLBACK_CLASS_NAMES`. + callback_class_name: Final = callback.__class__.__name__ + runs_on_no_log: Final = isinstance(callback, CustomLogger) and ( + callback_class_name.startswith("_PROXY_") or callback_class_name in _PROXY_INTERNAL_CALLBACK_CLASS_NAMES + ) + if not runs_on_no_log: verbose_logger.debug("no-log request, skipping logging for %s event", event_hook) return False diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 6801c8521e5..0a13e359635 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -356,7 +356,11 @@ class _ProxyDBLogger(CustomLogger): response_obj, start_time, end_time, - store_bodies=should_store_prompts_and_responses_in_spend_logs(), + store_bodies=should_store_prompts_and_responses_in_spend_logs( + cast( # cast-ok: kwargs mapping asserted for basedpyright; no runtime effect + "Mapping[str, object] | None", kwargs.get("litellm_params") + ) + ), ) if isinstance(event, SpendEventBuildError): verbose_proxy_logger.warning("collector: tracking cost in-process, event not buildable: %s", event.reason) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..aa38321b72b 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -207,6 +207,7 @@ def _stored_user_email(metadata: Mapping[str, object]) -> str | None: def _get_spend_logs_metadata( metadata: dict | None, + request_litellm_params: Mapping[str, object] | None = None, applied_guardrails: list[str] | None = None, batch_models: list[str] | None = None, batch_successful_requests: int | None = None, @@ -314,9 +315,11 @@ def _get_spend_logs_metadata( clean_metadata["batch_failed_requests"] = batch_failed_requests clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata clean_metadata["vector_store_request_metadata"] = _get_vector_store_request_for_spend_logs_payload( - vector_store_request_metadata + vector_store_request_metadata, request_litellm_params + ) + clean_metadata["guardrail_information"] = _sanitize_guardrail_information_for_spend_logs( + guardrail_information, request_litellm_params ) - clean_metadata["guardrail_information"] = _sanitize_guardrail_information_for_spend_logs(guardrail_information) clean_metadata["usage_object"] = usage_object clean_metadata["model_map_information"] = model_map_information clean_metadata["cold_storage_object_key"] = cold_storage_object_key @@ -738,6 +741,9 @@ def get_logging_payload( # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( persisted_metadata, + request_litellm_params=cast( # cast-ok: kwargs mapping asserted for basedpyright; no runtime effect + "Mapping[str, object] | None", litellm_params + ), applied_guardrails=( standard_logging_payload["metadata"].get("applied_guardrails", None) if standard_logging_payload is not None @@ -909,9 +915,19 @@ def get_logging_payload( requester_ip_address=clean_metadata.get("requester_ip_address", None), custom_llm_provider=custom_llm_provider or "", messages=_get_messages_for_spend_logs_payload( - standard_logging_payload=standard_logging_payload, metadata=metadata + standard_logging_payload=standard_logging_payload, + metadata=metadata, + request_litellm_params=cast( # cast-ok: kwargs mapping asserted for basedpyright; no runtime effect + "Mapping[str, object] | None", litellm_params + ), + ), + response=_get_response_for_spend_logs_payload( + payload=standard_logging_payload, + kwargs=kwargs, + request_litellm_params=cast( # cast-ok: kwargs mapping asserted for basedpyright; no runtime effect + "Mapping[str, object] | None", litellm_params + ), ), - response=_get_response_for_spend_logs_payload(payload=standard_logging_payload, kwargs=kwargs), proxy_server_request=_get_proxy_server_request_for_spend_logs_payload( metadata=metadata, litellm_params=( @@ -1196,8 +1212,9 @@ async def get_spend_by_team_and_customer( def _get_messages_for_spend_logs_payload( standard_logging_payload: StandardLoggingPayload | None, metadata: dict | None = None, + request_litellm_params: Mapping[str, object] | None = None, ) -> str: - if should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(request_litellm_params): if standard_logging_payload is not None: call_type: Final = standard_logging_payload.get("call_type", "") if call_type == "_arealtime": @@ -1456,6 +1473,7 @@ def _redact_prompt_leaks_in_error_string(text: str) -> str: def _sanitize_guardrail_information_for_spend_logs( guardrail_information: list[StandardLoggingGuardrailInformation] | None, + request_litellm_params: Mapping[str, object] | None = None, ) -> list[StandardLoggingGuardrailInformation] | None: """ When ``store_prompts_in_spend_logs`` is False, redact prompt-carrying fields @@ -1476,7 +1494,7 @@ def _sanitize_guardrail_information_for_spend_logs( here to match OTEL's defensive read pattern; otherwise iteration would yield the dict's keys and crash the whole spend-log write. """ - if guardrail_information is None or should_store_prompts_and_responses_in_spend_logs(): + if guardrail_information is None or should_store_prompts_and_responses_in_spend_logs(request_litellm_params): return guardrail_information entries: Final = [guardrail_information] if isinstance(guardrail_information, dict) else guardrail_information return [_redact_prompt_fields_in_guardrail_entry(entry) for entry in entries if isinstance(entry, dict)] @@ -1862,7 +1880,7 @@ def _get_proxy_server_request_for_spend_logs_payload( If turn_off_message_logging is enabled, redact messages in the request body. """ - if should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(litellm_params): _proxy_server_request: Final = cast(dict | None, litellm_params.get("proxy_server_request", EMPTY_MAPPING)) if _proxy_server_request is not None: _request_body = _proxy_server_request.get("body", EMPTY_MAPPING) or EMPTY_MAPPING @@ -1912,11 +1930,12 @@ def _get_proxy_server_request_for_spend_logs_payload( def _get_vector_store_request_for_spend_logs_payload( vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None, + request_litellm_params: Mapping[str, object] | None = None, ) -> list[StandardLoggingVectorStoreRequest] | None: """ If user does not want to store prompts and responses, then remove the content from the vector store request metadata """ - if should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(request_litellm_params): return vector_store_request_metadata # if user does not want to store prompts and responses, then remove the content from the vector store request metadata @@ -1937,10 +1956,11 @@ def _get_vector_store_request_for_spend_logs_payload( def _get_response_for_spend_logs_payload( payload: StandardLoggingPayload | None, kwargs: dict | None = None, + request_litellm_params: Mapping[str, object] | None = None, ) -> str: if payload is None: return "{}" - if should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(request_litellm_params): response_obj: object = payload.get("response") if response_obj is None: return "{}" @@ -1990,7 +2010,33 @@ def _get_response_for_spend_logs_payload( return "{}" -def should_store_prompts_and_responses_in_spend_logs() -> bool: +def _request_is_no_log(request_litellm_params: Mapping[str, object] | None) -> bool: + """`no-log: true` asks for this request's content not to be logged. + + The proxy still bills such a request -- that is why the spend writer is let through the + no-log filter in `should_run_callback` -- so this suppresses the content-bearing fields + only, never the cost fields. + + An administrator who set `litellm.global_disable_no_log_param` has turned the parameter off, + so it is ignored here exactly as `should_run_callback` ignores it. + """ + if litellm.global_disable_no_log_param: + return False + return isinstance(request_litellm_params, Mapping) and request_litellm_params.get("no-log") is True + + +def should_store_prompts_and_responses_in_spend_logs( + request_litellm_params: Mapping[str, object] | None = None, +) -> bool: + """Whether prompt/response content may be stored, for this request when one is given. + + A request that asked for `no-log: true` never has its content stored, whatever the + proxy-wide `store_prompts_in_spend_logs` setting says. Callers that have no per-request + context omit the argument and get the proxy-wide answer. + """ + if _request_is_no_log(request_litellm_params): + return False + from litellm.proxy.proxy_server import general_settings from litellm.secret_managers.main import get_secret_bool diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 8509b1d6f7d..0cdef84bac7 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -13566,3 +13566,113 @@ def test_legacy_s3_logs_a_streamed_anthropic_messages_call_handed_off_from_the_a assert s3_writer.log_event.call_count == expected_s3_writes if expected_s3_writes: assert s3_writer.log_event.call_args.kwargs["response_obj"] is complete_response +# --- no-log callback filtering (#44503) --- + + +class My_proxyAuditLogger(CustomLogger): + """Unrelated logger whose name merely contains "proxy" (Greptile's bypass example).""" + + def __init__(self) -> None: + super().__init__() + self.logged: list[object] = [] + + async def async_log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.logged.append(response_obj) + + +class _PROXY_HandlerStub(CustomLogger): + """Stands in for the proxy's internal `_PROXY_*` handler family.""" + + +async def _dispatch_async_success( + logging_obj: LitellmLogging, callbacks: list[CustomLogger], *, no_log: bool +) -> ModelResponse: + """Drive the real async success dispatch with `callbacks` as the combined callback list.""" + logging_obj.stream = False + logging_obj.model_call_details["litellm_params"] = {"acompletion": True, "no-log": no_log} + logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] + result: Final = _success_response() + with patch.object(logging_obj, "get_combined_callback_list", return_value=callbacks): + await logging_obj.async_success_handler( + result=result, + start_time=datetime.datetime.now(datetime.timezone.utc), + end_time=datetime.datetime.now(datetime.timezone.utc), + ) + return result + + +@pytest.mark.asyncio +async def test_no_log_request_still_runs_the_real_proxy_spend_logger( + logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch +): + """#44503: `no-log` must not stop the proxy from doing its own spend accounting. + + Drives the real `_ProxyDBLogger` through the real async success dispatch; only the + DB-writing `_PROXY_track_cost_callback` is recorded. + """ + from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger + + tracked: list[object] = [] + proxy_spend_logger: Final = _ProxyDBLogger() + + async def _record_spend( + kwargs: dict[str, object], + completion_response: object, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + ) -> None: + tracked.append(completion_response) + + monkeypatch.setattr(proxy_spend_logger, "_PROXY_track_cost_callback", _record_spend) + monkeypatch.setattr(litellm, "global_disable_no_log_param", False) + + result = await _dispatch_async_success(logging_obj, [proxy_spend_logger], no_log=True) + + assert len(tracked) == 1 and tracked[0] is result, "no-log request must still execute the spend accounting" + + +@pytest.mark.asyncio +async def test_no_log_suppresses_logger_whose_name_mentions_proxy( + logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch +): + """An unrelated logger named like a proxy hook must not bypass the no-log filter.""" + monkeypatch.setattr(litellm, "global_disable_no_log_param", False) + logger: Final = My_proxyAuditLogger() + + await _dispatch_async_success(logging_obj, [logger], no_log=True) + + assert logger.logged == [], "a name that merely contains 'proxy' must not bypass no-log" + + +@pytest.mark.asyncio +async def test_logger_whose_name_mentions_proxy_runs_without_no_log( + logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch +): + """Differential control: without `no-log`, the same logger runs (dispatch is not broken).""" + monkeypatch.setattr(litellm, "global_disable_no_log_param", False) + logger: Final = My_proxyAuditLogger() + + result = await _dispatch_async_success(logging_obj, [logger], no_log=False) + + assert len(logger.logged) == 1 and logger.logged[0] is result + + +def test_no_log_keeps_running_proxy_prefixed_handlers( + logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch +): + """The proxy's `_PROXY_*` handler family must keep running on no-log requests.""" + monkeypatch.setattr(litellm, "global_disable_no_log_param", False) + assert ( + logging_obj.should_run_callback( + callback=_PROXY_HandlerStub(), + litellm_params={"no-log": True}, + event_hook="success", + ) + is True + ) diff --git a/tests/unit/proxy/spend_tracking/test_no_log_request_spend_body_privacy.py b/tests/unit/proxy/spend_tracking/test_no_log_request_spend_body_privacy.py new file mode 100755 index 00000000000..d9616aa62f7 --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_no_log_request_spend_body_privacy.py @@ -0,0 +1,168 @@ +"""A `no-log: true` request is still billed, but its content must never be persisted or published. + +`_ProxyDBLogger` is deliberately let through the `should_run_callback` no-log filter so the proxy can +bill the request. These tests pin the other half of that contract: the spend row keeps the cost +fields and loses the prompt/response content, in both the DB write and the collector publish. +""" + +import datetime +import json +from datetime import timezone +from typing import Final + +import pytest + +import litellm +from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + +PROMPT_SECRET: Final = "PROMPT-SECRET-3f9a1c" +COMPLETION_SECRET: Final = "COMPLETION-SECRET-7b2d4e" + + +def _sl_object() -> dict: + return { + "messages": [{"role": "user", "content": PROMPT_SECRET}], + "response": {"choices": [{"message": {"role": "assistant", "content": COMPLETION_SECRET}}]}, + "metadata": {"user_api_key": "hashed-key"}, + "model_map_information": None, + "request_tags": [], + "response_cost": 0.001, + } + + +def _kwargs(no_log: bool) -> dict: + return { + "model": "gpt-4o-mini", + "call_type": "acompletion", + "litellm_params": { + "no-log": no_log, + "metadata": {"user_api_key": "hashed-key"}, + "proxy_server_request": { + "url": "http://localhost:4000/v1/chat/completions", + "body": {"messages": [{"role": "user", "content": PROMPT_SECRET}]}, + }, + }, + "standard_logging_object": _sl_object(), + "response_cost": 0.001, + } + + +def _response() -> litellm.ModelResponse: + return litellm.ModelResponse( + id="chatcmpl-no-log", + choices=[{"message": {"role": "assistant", "content": COMPLETION_SECRET}}], + usage=litellm.Usage(prompt_tokens=3, completion_tokens=4, total_tokens=7), + ) + + +def _payload(monkeypatch, no_log: bool, store_prompts: bool) -> dict: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": store_prompts}) + now: Final = datetime.datetime.now(timezone.utc) + return get_logging_payload( + kwargs=_kwargs(no_log=no_log), response_obj=_response(), start_time=now, end_time=now + ) + + +@pytest.mark.parametrize("store_prompts", [True, False]) +def test_no_log_request_persists_no_content(monkeypatch, store_prompts: bool): + payload: Final = _payload(monkeypatch, no_log=True, store_prompts=store_prompts) + stored: Final = json.dumps(payload, default=str) + + assert PROMPT_SECRET not in stored, "prompt leaked into the no-log spend row" + assert COMPLETION_SECRET not in stored, "completion leaked into the no-log spend row" + + +def test_no_log_request_still_records_cost(monkeypatch): + """The point of letting the spend writer through: the request is still billed.""" + payload: Final = _payload(monkeypatch, no_log=True, store_prompts=True) + + assert payload["request_id"] + assert payload["model"] == "gpt-4o-mini" + assert payload["total_tokens"] == 7 + assert payload["spend"] == 0.001 + + +def test_logged_request_still_persists_content_when_prompt_storage_is_on(monkeypatch): + """Control: the suppression is scoped to no-log, it is not a blanket redaction.""" + payload: Final = _payload(monkeypatch, no_log=False, store_prompts=True) + + # The request body carries the prompt for any call type; the SL `response` column carries the + # completion. (`messages` is only populated for `_arealtime` calls, so it is not asserted here.) + assert PROMPT_SECRET in payload["proxy_server_request"] + assert COMPLETION_SECRET in payload["response"] + + +class _CapturingProducer: + def __init__(self) -> None: + self.published: list[bytes] = [] + + async def publish(self, line: bytes) -> None: + self.published.append(line) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("no_log", [True, False]) +async def test_collector_event_carries_no_content_for_no_log_requests(monkeypatch, no_log: bool): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True}) + producer: Final = _CapturingProducer() + logger: Final = _ProxyDBLogger(spend_event_producer=producer) + now: Final = datetime.datetime.now(timezone.utc) + + await logger.async_log_success_event(_kwargs(no_log=no_log), _response(), now, now) + + assert producer.published, "expected the collector (offload) path to be taken" + published: Final = producer.published[0].decode() + if no_log: + assert PROMPT_SECRET not in published + assert COMPLETION_SECRET not in published + else: + assert PROMPT_SECRET in published + + +VECTOR_STORE_SECRET: Final = "VECTOR-STORE-SECRET-5c8e0a" + + +@pytest.mark.parametrize("global_disable_no_log_param", [False, True]) +@pytest.mark.parametrize("store_prompts", [True, False]) +@pytest.mark.parametrize("no_log", [True, False]) +def test_spend_row_content_follows_no_log_and_admin_settings( + monkeypatch, no_log: bool, store_prompts: bool, global_disable_no_log_param: bool +): + """Content is stored only when prompt storage is on and the request's `no-log` is honoured. + + `global_disable_no_log_param` lets an administrator turn the request parameter off; the + spend row must then follow `store_prompts_in_spend_logs` alone, as the callback filter does. + Cost is recorded in every combination. + """ + monkeypatch.setattr(litellm, "global_disable_no_log_param", global_disable_no_log_param) + payload: Final = _payload(monkeypatch, no_log=no_log, store_prompts=store_prompts) + content_expected: Final = store_prompts and not (no_log and not global_disable_no_log_param) + + assert (PROMPT_SECRET in payload["proxy_server_request"]) is content_expected + assert (COMPLETION_SECRET in payload["response"]) is content_expected + assert payload["spend"] == 0.001 + assert payload["total_tokens"] == 7 + + +@pytest.mark.parametrize("global_disable_no_log_param", [False, True]) +@pytest.mark.parametrize("no_log", [True, False]) +def test_vector_store_content_follows_request_no_log(monkeypatch, no_log: bool, global_disable_no_log_param: bool): + """Vector-store search results are request content too: a no-log row must not carry them.""" + from litellm.proxy import proxy_server + + monkeypatch.setattr(litellm, "global_disable_no_log_param", global_disable_no_log_param) + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True}) + kwargs: Final = _kwargs(no_log=no_log) + kwargs["standard_logging_object"]["metadata"]["vector_store_request_metadata"] = [ + {"vector_store_search_response": {"data": [{"content": [{"text": VECTOR_STORE_SECRET, "type": "text"}]}]}} + ] + now: Final = datetime.datetime.now(timezone.utc) + + payload: Final = get_logging_payload(kwargs=kwargs, response_obj=_response(), start_time=now, end_time=now) + + assert (VECTOR_STORE_SECRET in json.dumps(payload, default=str)) is (not no_log or global_disable_no_log_param)