This commit is contained in:
dingdangmao 2026-10-24 08:30:15 +00:00 • committed by GitHub
commit 1663930327
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 352 additions and 14 deletions

View file

@ -431,6 +431,10 @@ _INPUT_PRICING_KEY_PREFIXES: Final = (
_OUTPUT_PRICING_KEY_PREFIXES: Final = ("output_cost_per_token",)
_BATCH_PRICING_KEY_SUFFIX: Final = "_batches"
# Proxy-internal callbacks that must run even on `no-log` requests: the proxy still has to do
# its own spend accounting. The `_PROXY_` handler prefix marks most of them; the spend writer
# `_ProxyDBLogger` predates that prefix, so it is listed explicitly (exact names, no substring).
_PROXY_INTERNAL_CALLBACK_CLASS_NAMES: Final = frozenset({"_ProxyDBLogger"})
_NO_CARRIED_RATES: Final[Mapping[str, object]] = MappingProxyType({})
@ -2647,9 +2651,15 @@ class Logging(LiteLLMLoggingBaseClass):
return True
if litellm_params.get("no-log", False) is True:
# proxy cost tracking cal backs should run
if not (isinstance(callback, CustomLogger) and "_PROXY_" in callback.__class__.__name__):
# Proxy-internal callbacks must still run on `no-log` requests: the proxy's own
# spend accounting has to record the request. They use the internal `_PROXY_`
# handler prefix or -- for the spend writer, which predates that prefix -- an
# explicit entry in `_PROXY_INTERNAL_CALLBACK_CLASS_NAMES`.
callback_class_name: Final = callback.__class__.__name__
runs_on_no_log: Final = isinstance(callback, CustomLogger) and (
callback_class_name.startswith("_PROXY_") or callback_class_name in _PROXY_INTERNAL_CALLBACK_CLASS_NAMES
)
if not runs_on_no_log:
verbose_logger.debug("no-log request, skipping logging for %s event", event_hook)
return False

View file

@ -356,7 +356,11 @@ class _ProxyDBLogger(CustomLogger):
response_obj,
start_time,
end_time,
store_bodies=should_store_prompts_and_responses_in_spend_logs(),
store_bodies=should_store_prompts_and_responses_in_spend_logs(
cast( # cast-ok: kwargs mapping asserted for basedpyright; no runtime effect
"Mapping[str, object] | None", kwargs.get("litellm_params")
)
),
)
if isinstance(event, SpendEventBuildError):
verbose_proxy_logger.warning("collector: tracking cost in-process, event not buildable: %s", event.reason)

View file

@ -207,6 +207,7 @@ def _stored_user_email(metadata: Mapping[str, object]) -> str | None:
def _get_spend_logs_metadata(
metadata: dict | None,
request_litellm_params: Mapping[str, object] | None = None,
applied_guardrails: list[str] | None = None,
batch_models: list[str] | None = None,
batch_successful_requests: int | None = None,
@ -314,9 +315,11 @@ def _get_spend_logs_metadata(
clean_metadata["batch_failed_requests"] = batch_failed_requests
clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata
clean_metadata["vector_store_request_metadata"] = _get_vector_store_request_for_spend_logs_payload(
vector_store_request_metadata
vector_store_request_metadata, request_litellm_params
)
clean_metadata["guardrail_information"] = _sanitize_guardrail_information_for_spend_logs(
guardrail_information, request_litellm_params
)
clean_metadata["guardrail_information"] = _sanitize_guardrail_information_for_spend_logs(guardrail_information)
clean_metadata["usage_object"] = usage_object
clean_metadata["model_map_information"] = model_map_information
clean_metadata["cold_storage_object_key"] = cold_storage_object_key
@ -738,6 +741,9 @@ def get_logging_payload(
# clean up litellm metadata
clean_metadata = _get_spend_logs_metadata(
persisted_metadata,
request_litellm_params=cast( # cast-ok: kwargs mapping asserted for basedpyright; no runtime effect
"Mapping[str, object] | None", litellm_params
),
applied_guardrails=(
standard_logging_payload["metadata"].get("applied_guardrails", None)
if standard_logging_payload is not None
@ -909,9 +915,19 @@ def get_logging_payload(
requester_ip_address=clean_metadata.get("requester_ip_address", None),
custom_llm_provider=custom_llm_provider or "",
messages=_get_messages_for_spend_logs_payload(
standard_logging_payload=standard_logging_payload, metadata=metadata
standard_logging_payload=standard_logging_payload,
metadata=metadata,
request_litellm_params=cast( # cast-ok: kwargs mapping asserted for basedpyright; no runtime effect
"Mapping[str, object] | None", litellm_params
),
),
response=_get_response_for_spend_logs_payload(
payload=standard_logging_payload,
kwargs=kwargs,
request_litellm_params=cast( # cast-ok: kwargs mapping asserted for basedpyright; no runtime effect
"Mapping[str, object] | None", litellm_params
),
),
response=_get_response_for_spend_logs_payload(payload=standard_logging_payload, kwargs=kwargs),
proxy_server_request=_get_proxy_server_request_for_spend_logs_payload(
metadata=metadata,
litellm_params=(
@ -1196,8 +1212,9 @@ async def get_spend_by_team_and_customer(
def _get_messages_for_spend_logs_payload(
standard_logging_payload: StandardLoggingPayload | None,
metadata: dict | None = None,
request_litellm_params: Mapping[str, object] | None = None,
) -> str:
if should_store_prompts_and_responses_in_spend_logs():
if should_store_prompts_and_responses_in_spend_logs(request_litellm_params):
if standard_logging_payload is not None:
call_type: Final = standard_logging_payload.get("call_type", "")
if call_type == "_arealtime":
@ -1456,6 +1473,7 @@ def _redact_prompt_leaks_in_error_string(text: str) -> str:
def _sanitize_guardrail_information_for_spend_logs(
guardrail_information: list[StandardLoggingGuardrailInformation] | None,
request_litellm_params: Mapping[str, object] | None = None,
) -> list[StandardLoggingGuardrailInformation] | None:
"""
When ``store_prompts_in_spend_logs`` is False, redact prompt-carrying fields
@ -1476,7 +1494,7 @@ def _sanitize_guardrail_information_for_spend_logs(
here to match OTEL's defensive read pattern; otherwise iteration would
yield the dict's keys and crash the whole spend-log write.
"""
if guardrail_information is None or should_store_prompts_and_responses_in_spend_logs():
if guardrail_information is None or should_store_prompts_and_responses_in_spend_logs(request_litellm_params):
return guardrail_information
entries: Final = [guardrail_information] if isinstance(guardrail_information, dict) else guardrail_information
return [_redact_prompt_fields_in_guardrail_entry(entry) for entry in entries if isinstance(entry, dict)]
@ -1862,7 +1880,7 @@ def _get_proxy_server_request_for_spend_logs_payload(
If turn_off_message_logging is enabled, redact messages in the request body.
"""
if should_store_prompts_and_responses_in_spend_logs():
if should_store_prompts_and_responses_in_spend_logs(litellm_params):
_proxy_server_request: Final = cast(dict | None, litellm_params.get("proxy_server_request", EMPTY_MAPPING))
if _proxy_server_request is not None:
_request_body = _proxy_server_request.get("body", EMPTY_MAPPING) or EMPTY_MAPPING
@ -1912,11 +1930,12 @@ def _get_proxy_server_request_for_spend_logs_payload(
def _get_vector_store_request_for_spend_logs_payload(
vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None,
request_litellm_params: Mapping[str, object] | None = None,
) -> list[StandardLoggingVectorStoreRequest] | None:
"""
If user does not want to store prompts and responses, then remove the content from the vector store request metadata
"""
if should_store_prompts_and_responses_in_spend_logs():
if should_store_prompts_and_responses_in_spend_logs(request_litellm_params):
return vector_store_request_metadata
# if user does not want to store prompts and responses, then remove the content from the vector store request metadata
@ -1937,10 +1956,11 @@ def _get_vector_store_request_for_spend_logs_payload(
def _get_response_for_spend_logs_payload(
payload: StandardLoggingPayload | None,
kwargs: dict | None = None,
request_litellm_params: Mapping[str, object] | None = None,
) -> str:
if payload is None:
return "{}"
if should_store_prompts_and_responses_in_spend_logs():
if should_store_prompts_and_responses_in_spend_logs(request_litellm_params):
response_obj: object = payload.get("response")
if response_obj is None:
return "{}"
@ -1990,7 +2010,33 @@ def _get_response_for_spend_logs_payload(
return "{}"
def should_store_prompts_and_responses_in_spend_logs() -> bool:
def _request_is_no_log(request_litellm_params: Mapping[str, object] | None) -> bool:
"""`no-log: true` asks for this request's content not to be logged.
The proxy still bills such a request -- that is why the spend writer is let through the
no-log filter in `should_run_callback` -- so this suppresses the content-bearing fields
only, never the cost fields.
An administrator who set `litellm.global_disable_no_log_param` has turned the parameter off,
so it is ignored here exactly as `should_run_callback` ignores it.
"""
if litellm.global_disable_no_log_param:
return False
return isinstance(request_litellm_params, Mapping) and request_litellm_params.get("no-log") is True
def should_store_prompts_and_responses_in_spend_logs(
request_litellm_params: Mapping[str, object] | None = None,
) -> bool:
"""Whether prompt/response content may be stored, for this request when one is given.
A request that asked for `no-log: true` never has its content stored, whatever the
proxy-wide `store_prompts_in_spend_logs` setting says. Callers that have no per-request
context omit the argument and get the proxy-wide answer.
"""
if _request_is_no_log(request_litellm_params):
return False
from litellm.proxy.proxy_server import general_settings
from litellm.secret_managers.main import get_secret_bool

View file

@ -13566,3 +13566,113 @@ def test_legacy_s3_logs_a_streamed_anthropic_messages_call_handed_off_from_the_a
assert s3_writer.log_event.call_count == expected_s3_writes
if expected_s3_writes:
assert s3_writer.log_event.call_args.kwargs["response_obj"] is complete_response
# --- no-log callback filtering (#44503) ---
class My_proxyAuditLogger(CustomLogger):
"""Unrelated logger whose name merely contains "proxy" (Greptile's bypass example)."""
def __init__(self) -> None:
super().__init__()
self.logged: list[object] = []
async def async_log_success_event(
self,
kwargs: dict[str, object],
response_obj: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None:
self.logged.append(response_obj)
class _PROXY_HandlerStub(CustomLogger):
"""Stands in for the proxy's internal `_PROXY_*` handler family."""
async def _dispatch_async_success(
logging_obj: LitellmLogging, callbacks: list[CustomLogger], *, no_log: bool
) -> ModelResponse:
"""Drive the real async success dispatch with `callbacks` as the combined callback list."""
logging_obj.stream = False
logging_obj.model_call_details["litellm_params"] = {"acompletion": True, "no-log": no_log}
logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"]
result: Final = _success_response()
with patch.object(logging_obj, "get_combined_callback_list", return_value=callbacks):
await logging_obj.async_success_handler(
result=result,
start_time=datetime.datetime.now(datetime.timezone.utc),
end_time=datetime.datetime.now(datetime.timezone.utc),
)
return result
@pytest.mark.asyncio
async def test_no_log_request_still_runs_the_real_proxy_spend_logger(
logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch
):
"""#44503: `no-log` must not stop the proxy from doing its own spend accounting.
Drives the real `_ProxyDBLogger` through the real async success dispatch; only the
DB-writing `_PROXY_track_cost_callback` is recorded.
"""
from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger
tracked: list[object] = []
proxy_spend_logger: Final = _ProxyDBLogger()
async def _record_spend(
kwargs: dict[str, object],
completion_response: object,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
) -> None:
tracked.append(completion_response)
monkeypatch.setattr(proxy_spend_logger, "_PROXY_track_cost_callback", _record_spend)
monkeypatch.setattr(litellm, "global_disable_no_log_param", False)
result = await _dispatch_async_success(logging_obj, [proxy_spend_logger], no_log=True)
assert len(tracked) == 1 and tracked[0] is result, "no-log request must still execute the spend accounting"
@pytest.mark.asyncio
async def test_no_log_suppresses_logger_whose_name_mentions_proxy(
logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch
):
"""An unrelated logger named like a proxy hook must not bypass the no-log filter."""
monkeypatch.setattr(litellm, "global_disable_no_log_param", False)
logger: Final = My_proxyAuditLogger()
await _dispatch_async_success(logging_obj, [logger], no_log=True)
assert logger.logged == [], "a name that merely contains 'proxy' must not bypass no-log"
@pytest.mark.asyncio
async def test_logger_whose_name_mentions_proxy_runs_without_no_log(
logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch
):
"""Differential control: without `no-log`, the same logger runs (dispatch is not broken)."""
monkeypatch.setattr(litellm, "global_disable_no_log_param", False)
logger: Final = My_proxyAuditLogger()
result = await _dispatch_async_success(logging_obj, [logger], no_log=False)
assert len(logger.logged) == 1 and logger.logged[0] is result
def test_no_log_keeps_running_proxy_prefixed_handlers(
logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch
):
"""The proxy's `_PROXY_*` handler family must keep running on no-log requests."""
monkeypatch.setattr(litellm, "global_disable_no_log_param", False)
assert (
logging_obj.should_run_callback(
callback=_PROXY_HandlerStub(),
litellm_params={"no-log": True},
event_hook="success",
)
is True
)

View file

@ -0,0 +1,168 @@
"""A `no-log: true` request is still billed, but its content must never be persisted or published.
`_ProxyDBLogger` is deliberately let through the `should_run_callback` no-log filter so the proxy can
bill the request. These tests pin the other half of that contract: the spend row keeps the cost
fields and loses the prompt/response content, in both the DB write and the collector publish.
"""
import datetime
import json
from datetime import timezone
from typing import Final
import pytest
import litellm
from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
PROMPT_SECRET: Final = "PROMPT-SECRET-3f9a1c"
COMPLETION_SECRET: Final = "COMPLETION-SECRET-7b2d4e"
def _sl_object() -> dict:
return {
"messages": [{"role": "user", "content": PROMPT_SECRET}],
"response": {"choices": [{"message": {"role": "assistant", "content": COMPLETION_SECRET}}]},
"metadata": {"user_api_key": "hashed-key"},
"model_map_information": None,
"request_tags": [],
"response_cost": 0.001,
}
def _kwargs(no_log: bool) -> dict:
return {
"model": "gpt-4o-mini",
"call_type": "acompletion",
"litellm_params": {
"no-log": no_log,
"metadata": {"user_api_key": "hashed-key"},
"proxy_server_request": {
"url": "http://localhost:4000/v1/chat/completions",
"body": {"messages": [{"role": "user", "content": PROMPT_SECRET}]},
},
},
"standard_logging_object": _sl_object(),
"response_cost": 0.001,
}
def _response() -> litellm.ModelResponse:
return litellm.ModelResponse(
id="chatcmpl-no-log",
choices=[{"message": {"role": "assistant", "content": COMPLETION_SECRET}}],
usage=litellm.Usage(prompt_tokens=3, completion_tokens=4, total_tokens=7),
)
def _payload(monkeypatch, no_log: bool, store_prompts: bool) -> dict:
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": store_prompts})
now: Final = datetime.datetime.now(timezone.utc)
return get_logging_payload(
kwargs=_kwargs(no_log=no_log), response_obj=_response(), start_time=now, end_time=now
)
@pytest.mark.parametrize("store_prompts", [True, False])
def test_no_log_request_persists_no_content(monkeypatch, store_prompts: bool):
payload: Final = _payload(monkeypatch, no_log=True, store_prompts=store_prompts)
stored: Final = json.dumps(payload, default=str)
assert PROMPT_SECRET not in stored, "prompt leaked into the no-log spend row"
assert COMPLETION_SECRET not in stored, "completion leaked into the no-log spend row"
def test_no_log_request_still_records_cost(monkeypatch):
"""The point of letting the spend writer through: the request is still billed."""
payload: Final = _payload(monkeypatch, no_log=True, store_prompts=True)
assert payload["request_id"]
assert payload["model"] == "gpt-4o-mini"
assert payload["total_tokens"] == 7
assert payload["spend"] == 0.001
def test_logged_request_still_persists_content_when_prompt_storage_is_on(monkeypatch):
"""Control: the suppression is scoped to no-log, it is not a blanket redaction."""
payload: Final = _payload(monkeypatch, no_log=False, store_prompts=True)
# The request body carries the prompt for any call type; the SL `response` column carries the
# completion. (`messages` is only populated for `_arealtime` calls, so it is not asserted here.)
assert PROMPT_SECRET in payload["proxy_server_request"]
assert COMPLETION_SECRET in payload["response"]
class _CapturingProducer:
def __init__(self) -> None:
self.published: list[bytes] = []
async def publish(self, line: bytes) -> None:
self.published.append(line)
@pytest.mark.asyncio
@pytest.mark.parametrize("no_log", [True, False])
async def test_collector_event_carries_no_content_for_no_log_requests(monkeypatch, no_log: bool):
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
producer: Final = _CapturingProducer()
logger: Final = _ProxyDBLogger(spend_event_producer=producer)
now: Final = datetime.datetime.now(timezone.utc)
await logger.async_log_success_event(_kwargs(no_log=no_log), _response(), now, now)
assert producer.published, "expected the collector (offload) path to be taken"
published: Final = producer.published[0].decode()
if no_log:
assert PROMPT_SECRET not in published
assert COMPLETION_SECRET not in published
else:
assert PROMPT_SECRET in published
VECTOR_STORE_SECRET: Final = "VECTOR-STORE-SECRET-5c8e0a"
@pytest.mark.parametrize("global_disable_no_log_param", [False, True])
@pytest.mark.parametrize("store_prompts", [True, False])
@pytest.mark.parametrize("no_log", [True, False])
def test_spend_row_content_follows_no_log_and_admin_settings(
monkeypatch, no_log: bool, store_prompts: bool, global_disable_no_log_param: bool
):
"""Content is stored only when prompt storage is on and the request's `no-log` is honoured.
`global_disable_no_log_param` lets an administrator turn the request parameter off; the
spend row must then follow `store_prompts_in_spend_logs` alone, as the callback filter does.
Cost is recorded in every combination.
"""
monkeypatch.setattr(litellm, "global_disable_no_log_param", global_disable_no_log_param)
payload: Final = _payload(monkeypatch, no_log=no_log, store_prompts=store_prompts)
content_expected: Final = store_prompts and not (no_log and not global_disable_no_log_param)
assert (PROMPT_SECRET in payload["proxy_server_request"]) is content_expected
assert (COMPLETION_SECRET in payload["response"]) is content_expected
assert payload["spend"] == 0.001
assert payload["total_tokens"] == 7
@pytest.mark.parametrize("global_disable_no_log_param", [False, True])
@pytest.mark.parametrize("no_log", [True, False])
def test_vector_store_content_follows_request_no_log(monkeypatch, no_log: bool, global_disable_no_log_param: bool):
"""Vector-store search results are request content too: a no-log row must not carry them."""
from litellm.proxy import proxy_server
monkeypatch.setattr(litellm, "global_disable_no_log_param", global_disable_no_log_param)
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
kwargs: Final = _kwargs(no_log=no_log)
kwargs["standard_logging_object"]["metadata"]["vector_store_request_metadata"] = [
{"vector_store_search_response": {"data": [{"content": [{"text": VECTOR_STORE_SECRET, "type": "text"}]}]}}
]
now: Final = datetime.datetime.now(timezone.utc)
payload: Final = get_logging_payload(kwargs=kwargs, response_obj=_response(), start_time=now, end_time=now)
assert (VECTOR_STORE_SECRET in json.dumps(payload, default=str)) is (not no_log or global_disable_no_log_param)