feat(router): log classifier input and masked source request

This commit is contained in:
moe-berri 2026-09-10 11:49:57 -07:00
parent 6c69dd0f72
commit 153c962e15
20 changed files with 673 additions and 38 deletions

View file

@ -927,6 +927,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
# Handle turn_off_message_logging - redact messages and responses (if not already excluded)
if turn_off_message_logging:
from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit
for field in CLASSIFIER_AUDIT_FIELDS:
standard_logging_object_copy.pop(field, None)
params: Final = model_call_details_copy.get("litellm_params")
request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None
if isinstance(params, dict) and isinstance(request, dict):
model_call_details_copy["litellm_params"] = {
**params,
"proxy_server_request": without_classifier_audit(request),
}
redacted_str: Final = "redacted-by-litellm"
if "messages" not in (excluded_fields or []) and standard_logging_object_copy.get("messages") is not None:

View file

@ -0,0 +1,59 @@
from collections.abc import Mapping
from typing import Final
from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ClassifierAudit
CLASSIFIER_AUDIT_FIELDS: Final = ("classifier_input", "originating_request_masked")
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def classifier_input_snapshot(value: object, *, openai_sdk: bool = False) -> Mapping[str, JsonValue] | None:
if openai_sdk and isinstance(value, Mapping):
body: Final = {
key: item for key, item in value.items() if key not in ("extra_headers", "extra_query", "extra_body")
}
extra_body: Final = value.get("extra_body")
return classifier_input_snapshot({**body, **extra_body} if isinstance(extra_body, Mapping) else body)
try:
return (
_JSON_OBJECT.validate_json(value)
if isinstance(value, (str, bytes))
else _JSON_OBJECT.validate_python(value)
)
except ValidationError:
return None
def is_classifier_call(call_type: str, params: Mapping[str, object]) -> bool:
return call_type in ("completion", "acompletion") and any(
isinstance(metadata := params.get(key), Mapping)
and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == AUTOROUTER_CLASSIFIER_CALL_ORIGIN
for key in ("metadata", "litellm_metadata")
)
def masked_originating_request(request_kwargs: Mapping[str, object] | None) -> Mapping[str, JsonValue] | None:
request: Final = (request_kwargs or {}).get("proxy_server_request")
body: Final = request.get("body") if isinstance(request, Mapping) else None
if not isinstance(body, Mapping):
return None
serializable: Final = classifier_input_snapshot(safe_dumps(body))
return classifier_input_snapshot(redact_credentials_in_payload(serializable)) if serializable is not None else None
def classifier_audit_fields(payload: Mapping[str, object]) -> ClassifierAudit:
classifier_input: Final = classifier_input_snapshot(payload.get("classifier_input"))
originating_request: Final = classifier_input_snapshot(payload.get("originating_request_masked"))
return {
**(ClassifierAudit(classifier_input=classifier_input) if classifier_input is not None else {}),
**(ClassifierAudit(originating_request_masked=originating_request) if originating_request is not None else {}),
}
def without_classifier_audit(payload: Mapping[str, object]) -> dict[str, object]:
return {key: value for key, value in payload.items() if key not in CLASSIFIER_AUDIT_FIELDS}

View file

@ -17,7 +17,7 @@ from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast
from httpx import Response
from pydantic import BaseModel
from pydantic import BaseModel, JsonValue
import litellm
from litellm import (
@ -64,6 +64,11 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.classifier_logging import (
classifier_audit_fields,
classifier_input_snapshot,
is_classifier_call,
)
from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.internal_call_metadata import (
@ -89,6 +94,7 @@ from litellm.litellm_core_utils.redact_messages import (
redact_message_input_output_from_custom_logger,
redact_message_input_output_from_logging,
redact_streaming_responses_for_custom_logger,
should_redact_message_logging,
)
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.llms.base_llm.search.transformation import SearchResponse
@ -473,6 +479,7 @@ class Logging(LiteLLMLoggingBaseClass):
stream_options = None
litellm_request_debug: bool = False
streamed_anthropic_message_id: str | None = None
classifier_input: Mapping[str, JsonValue] | None = None
def __init__(
self,
@ -1211,6 +1218,14 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["api_key"] = api_key
self.model_call_details["additional_args"] = additional_args
self.model_call_details["log_event_type"] = "pre_api_call"
if is_classifier_call(self.call_type, self.model_call_details.get("litellm_params") or {}):
self.classifier_input = (
None
if should_redact_message_logging(self.model_call_details)
else classifier_input_snapshot(
additional_args.get("complete_input_dict"), openai_sdk=additional_args.get("openai_sdk") is True
)
)
if model: # if model name was changes pre-call, overwrite the initial model call name with the new one
self.model_call_details["model"] = model
self.model_call_details["litellm_params"]["api_base"] = self._get_masked_api_base(
@ -6293,6 +6308,16 @@ def get_standard_logging_object_payload(
)
payload: Final[StandardLoggingPayload] = StandardLoggingPayload(
**(
classifier_audit_fields(
{
"classifier_input": logging_obj.classifier_input,
"originating_request_masked": proxy_server_request.get("originating_request_masked"),
}
)
if is_classifier_call(call_type or "", litellm_params) and not should_redact_message_logging(kwargs)
else {}
),
id=str(id),
litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),
trace_id=StandardLoggingPayloadSetup.get_standard_logging_payload_trace_id(

View file

@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm.constants import REDACTED_BY_LITELLM
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit
from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
)
@ -183,6 +184,9 @@ def _redact_standard_logging_object(model_call_details: dict):
redacted_str: Final = REDACTED_BY_LITELLM
for field in CLASSIFIER_AUDIT_FIELDS:
standard_logging_object.pop(field, None)
if standard_logging_object.get("messages") is not None:
standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}]
@ -254,6 +258,12 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
copy via redact_streaming_responses_for_custom_logger instead.
"""
# Redact model_call_details
for field in CLASSIFIER_AUDIT_FIELDS:
model_call_details.pop(field, None)
params: Final = model_call_details.get("litellm_params")
request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None
if isinstance(params, dict) and isinstance(request, Mapping):
model_call_details["litellm_params"] = {**params, "proxy_server_request": without_classifier_audit(request)}
model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
model_call_details["prompt"] = ""
model_call_details["input"] = ""

View file

@ -797,6 +797,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
"api_base": openai_client._base_url._uri_reference,
"acompletion": acompletion,
"complete_input_dict": data,
"openai_sdk": True,
},
)
@ -938,6 +939,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
"api_base": openai_aclient._base_url._uri_reference,
"acompletion": True,
"complete_input_dict": data,
"openai_sdk": True,
},
)

View file

@ -26,7 +26,8 @@ from typing_extensions import ReadOnly
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME
from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields, classifier_input_snapshot
from litellm.proxy._types import *
from litellm.proxy._types import ProviderBudgetResponse, ProviderBudgetResponseObject
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@ -3099,7 +3100,11 @@ async def _resolve_request_response_payload(
proxy_server_request: Final = row.get("proxy_server_request")
pg_payload: Final = RequestResponsePayload(messages, response, proxy_server_request)
if (
stored_request: Final = classifier_input_snapshot(proxy_server_request)
truncated_audit: Final = bool(stored_request and classifier_audit_fields(stored_request)) and (
LITELLM_TRUNCATED_PAYLOAD_FIELD in str(proxy_server_request)
)
if not truncated_audit and (
_spend_log_field_has_content(messages)
or _spend_log_field_has_content(response)
or _spend_log_field_has_content(proxy_server_request)
@ -3124,10 +3129,19 @@ async def _resolve_request_response_payload(
if payload is None:
return pg_payload
cold_audit: Final = classifier_audit_fields(payload)
resolved_request: Final = (
{**(classifier_input_snapshot(payload.get("proxy_server_request")) or stored_request or {}), **cold_audit}
if cold_audit
else payload.get("proxy_server_request")
)
if truncated_audit:
return RequestResponsePayload(messages, response, resolved_request if cold_audit else proxy_server_request)
return RequestResponsePayload(
messages=payload.get("messages"),
response=payload.get("response"),
proxy_server_request=payload.get("proxy_server_request"),
proxy_server_request=resolved_request,
)

View file

@ -22,6 +22,7 @@ from litellm.constants import (
from litellm.constants import (
MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB,
)
from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields
from litellm.litellm_core_utils.core_helpers import (
get_litellm_metadata_from_kwargs,
reconstruct_model_name,
@ -1251,6 +1252,10 @@ def _get_proxy_server_request_for_spend_logs_payload(
if _proxy_server_request is not None:
_request_body = _proxy_server_request.get("body", {}) or {}
standard_payload: Final = (kwargs or {}).get("standard_logging_object")
if isinstance(standard_payload, Mapping):
_request_body = {**_request_body, **classifier_audit_fields(standard_payload)}
if kwargs is not None:
realtime_tools: Final = kwargs.get("realtime_tools")
if realtime_tools:

View file

@ -36,6 +36,7 @@ from litellm.constants import (
SESSION_ID_GENERATED_METADATA_KEY,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.classifier_logging import masked_originating_request
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
get_metadata_variable_name_from_kwargs,
@ -1924,11 +1925,8 @@ class ComplexityRouter(CustomLogger):
Call the configured classifier model with a system/user role split and prior-turn context.
Builds a structured classification prompt with:
- System message: the stable classifier rubric AND the caller's own system prompt (task
constraints). This is the largest, most repeated part of the call, so keeping it in the
system role lets the provider prompt-cache it across a session's classifier calls.
- User message: the variable payload -- a few prior user turns for context and the current
ask to classify.
- System message: the stable classifier rubric.
- User message: the caller's system prompt quoted as task context, prior turns, and the current ask.
Args:
prompt: The current user ask text (already extracted as the real human ask, not tool results)
@ -2005,12 +2003,13 @@ class ComplexityRouter(CustomLogger):
classifier_call_params = MappingProxyType({"reasoning_effort": llm_config.reasoning_effort})
proxy_server_request: Final = {
"originating_request_masked": masked_originating_request(request_kwargs),
"body": {
"model": llm_config.model,
"messages": messages_for_call,
"response_format": response_format,
**classifier_call_params,
}
},
}
classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000

View file

@ -35,6 +35,7 @@ from pydantic import (
BaseModel,
ConfigDict,
Field,
JsonValue,
PrivateAttr,
SkipValidation,
field_serializer,
@ -3373,7 +3374,12 @@ class StandardAuditLogPayload(TypedDict):
updated_values: str | None
class StandardLoggingPayload(TypedDict):
class ClassifierAudit(TypedDict, total=False):
classifier_input: ReadOnly[Mapping[str, JsonValue]]
originating_request_masked: ReadOnly[Mapping[str, JsonValue]]
class StandardLoggingPayload(ClassifierAudit):
id: str
trace_id: str # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries)
session_id: str # End-user/conversation session id (litellm_session_id), independent of trace_id

View file

@ -1,4 +1,5 @@
# Reliability & Performance (behavior features). Grounded in litellm/router.py + router_strategy/ + router_utils/.
- {id: reliability.routing.classifier_audit.separates_provider_input_and_source, module: reliability, tier: P1, behavior: routing, variant: classifier_audit, assertions: [separates_provider_input_and_source], exercised_on: [chat_completions, messages, responses], source: "litellm/litellm_core_utils/classifier_logging.py", rationale: "Classifier spend details distinguish provider input from the credential-masked originating request across all three request surfaces"}
- {id: reliability.fallback.5xx.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: "5xx", assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:2024", rationale: "Reroute on provider 5xx to alternate deployment"}
- {id: reliability.fallback.context_window.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: context_window, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6108", rationale: "Fallback when model exceeds context limit"}
- {id: reliability.fallback.content_policy.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: content_policy, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6023", rationale: "Reroute on content-policy violation"}

View file

@ -0,0 +1,143 @@
import json
import os
from collections.abc import Iterator
from contextlib import ExitStack
from dataclasses import dataclass
from typing import Final, Literal
import pytest
from pydantic import BaseModel, Field, JsonValue, TypeAdapter
from e2e_config import unique_marker
from e2e_http import AnthropicHeaders, NoBody, unwrap
from models import ChatMessage, KeyGenerateBody, LiteLLMParamsBody
from proxy_client import ProxyClient
pytestmark = pytest.mark.e2e
class AuditMetadata(BaseModel):
source_marker: str
authorization: str = "synthetic-audit-secret"
class AuditHeaders(AnthropicHeaders):
enable_redaction: str | None = Field(default=None, serialization_alias="x-litellm-enable-message-redaction")
class AuditRequest(BaseModel):
model: str
messages: list[ChatMessage] | None = None
system: str | None = None
instructions: str | None = None
input: str | None = None
max_tokens: int | None = None
max_output_tokens: int | None = None
metadata: AuditMetadata | None = None
litellm_metadata: AuditMetadata | None = None
class AuditResponse(BaseModel):
id: str
class AuditDetail(BaseModel):
proxy_server_request: dict[str, JsonValue] | str | None = None
response: dict[str, JsonValue] | str | None = None
@dataclass(frozen=True, slots=True)
class AuditDeployment:
alias: str
key: str
@pytest.fixture
def audit_deployment(proxy: ProxyClient, provider: str) -> Iterator[AuditDeployment]:
marker: Final = unique_marker()
classifier: Final = f"audit-classifier-{marker}"
target: Final = f"audit-target-{marker}"
alias: Final = f"audit-router-{marker}"
model: Final = os.environ.get(
f"E2E_CHEAP_{provider.upper()}_MODEL", "gpt-5.6" if provider == "openai" else "claude-haiku-4-5"
)
params: Final = LiteLLMParamsBody(
model=f"{provider}/{model}",
api_key=os.environ.get(f"{provider.upper()}_API_KEY") or f"os.environ/{provider.upper()}_API_KEY",
api_base=os.environ.get(f"{provider.upper()}_API_BASE"),
)
with ExitStack() as stack:
for name in (classifier, target):
stack.callback(proxy.delete_model, proxy.create_model(name, params))
stack.callback(proxy.delete_model, proxy.create_model(alias, LiteLLMParamsBody(
model="auto_router/complexity_router",
complexity_router_config={
"classifier_type": "llm",
"classifier_llm_config": {"model": classifier, "timeout_ms": 30000},
"tiers": {tier: target for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")},
},
)))
key: Final = proxy.generate_key(KeyGenerateBody(models=[alias, classifier, target]))
stack.callback(proxy.delete_key, key)
yield AuditDeployment(alias, key)
class TestClassifierAudit:
@pytest.mark.covers(
"reliability.routing.classifier_audit.separates_provider_input_and_source",
exercised_on=("chat_completions", "messages", "responses"),
)
@pytest.mark.parametrize("surface", ["chat_completions", "messages", "responses"])
@pytest.mark.parametrize("provider", ["openai", "anthropic"])
@pytest.mark.parametrize("redact", [False, True])
def test_classifier_audit_separates_input_and_masked_source(
self, proxy: ProxyClient, audit_deployment: AuditDeployment, surface: Literal["chat_completions", "messages", "responses"],
redact: bool,
) -> None:
marker: Final = unique_marker()
source_marker: Final = f"source-only-{marker}"
prompt: Final = f"Reply with hello. Request label {marker}"
metadata: Final = AuditMetadata(source_marker=source_marker)
body: Final = AuditRequest(
model=audit_deployment.alias,
messages=[ChatMessage(role="user", content=prompt)] if surface != "responses" else None,
input=prompt if surface == "responses" else None,
system="Be concise" if surface == "messages" else None,
instructions="Be concise" if surface == "responses" else None,
max_tokens=128 if surface != "responses" else None,
max_output_tokens=128 if surface == "responses" else None,
metadata=metadata if surface != "responses" else None,
litellm_metadata=metadata if surface == "responses" else None,
)
path: Final = {"chat_completions": "/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}[surface]
response: Final = unwrap(proxy.transport.post(
path, headers=AuditHeaders(authorization=f"Bearer {audit_deployment.key}", enable_redaction="true" if redact else None),
json=body, response_type=AuditResponse,
))
assert response.id
rows: Final = proxy.poll_logs_for_key(audit_deployment.key, min_rows=2)
assert len(rows) == 2, "Expected a classifier spend row and a routed response spend row"
details: Final = tuple(
unwrap(proxy.transport.get(
f"/spend/logs/ui/{row.request_id}", headers=proxy.transport.master,
params=NoBody(), response_type=AuditDetail,
)) for row in rows
)
adapter: Final = TypeAdapter(dict[str, JsonValue])
requests: Final = tuple(
adapter.validate_json(detail.proxy_server_request) if isinstance(detail.proxy_server_request, str)
else detail.proxy_server_request or {} for detail in details
)
audits: Final = tuple(item for item in requests if "classifier_input" in item)
if redact:
assert audits == ()
assert all("originating_request_masked" not in item for item in requests)
return
assert len(audits) == 1, "The audit belongs only to the classifier call"
audit: Final = audits[0]
assert marker in json.dumps(audit["classifier_input"])
assert source_marker not in json.dumps(audit["classifier_input"])
assert source_marker in json.dumps(audit["originating_request_masked"])
assert "synthetic-audit-secret" not in json.dumps(audit)
assert audit_deployment.alias in json.dumps(audit["originating_request_masked"])
assert any("tier" in str(detail.response) for detail in details)

View file

@ -0,0 +1,38 @@
from typing import Final
import pytest
from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot, masked_originating_request
@pytest.mark.parametrize("encoded", [False, True])
def test_classifier_snapshot_preserves_provider_shape_and_is_independent(encoded: bool) -> None:
import json
provider_body: Final = {"system": [{"text": "rubric"}], "messages": [{"role": "user", "content": "ask"}]}
snapshot: Final = classifier_input_snapshot(json.dumps(provider_body) if encoded else provider_body)
assert snapshot == provider_body
provider_body["messages"][0]["content"] = "later mutation"
assert snapshot == {"system": [{"text": "rubric"}], "messages": [{"role": "user", "content": "ask"}]}
def test_originating_snapshot_masks_nested_credentials_without_altering_source() -> None:
body: Final = {
"model": "router",
"input": [{"type": "message", "role": "user", "content": "source-only"}],
"api_key": "short",
"metadata": {"nested": [{"Authorization": "Bearer secret", "access_token": 123}]},
}
snapshot: Final = masked_originating_request({"proxy_server_request": {"body": body}})
assert snapshot is not None
assert snapshot["model"] == "router"
assert snapshot["input"] == body["input"]
assert snapshot["api_key"] == "REDACTED"
assert snapshot["metadata"] == {"nested": [{"Authorization": "REDACTED", "access_token": "REDACTED"}]}
assert body["api_key"] == "short"
assert body["metadata"]["nested"][0]["Authorization"] == "Bearer secret"
@pytest.mark.parametrize("value", [None, "not-json", [], {"messages": object()}])
def test_invalid_provider_payload_is_not_reported_as_captured(value: object) -> None:
assert classifier_input_snapshot(value) is None

View file

@ -3,7 +3,7 @@ import contextlib
import datetime
import os
import sys
from typing import Literal
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -6801,3 +6801,122 @@ def test_get_error_information_redacts_provider_key_from_upstream_url():
assert "REDACTED" in result["traceback"]
assert "REDACTED" in result["error_message"]
assert result["error_code"] == "400"
@pytest.mark.asyncio
@pytest.mark.parametrize("provider", ["openai", "anthropic", "bedrock"])
async def test_classifier_audit_matches_provider_transport(provider: str) -> None:
import json
from openai import AsyncOpenAI
from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
outbound: Final = asyncio.Queue()
logs: Final = asyncio.Queue()
def respond(request: httpx.Request) -> httpx.Response:
outbound.put_nowait(json.loads(request.content))
content: Final = '{"tier":"SIMPLE"}'
if provider == "anthropic":
return httpx.Response(200, json={
"id": "msg-audit", "type": "message", "role": "assistant", "model": "claude-haiku-4-5",
"content": [{"type": "text", "text": content}], "stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 5},
})
if provider == "bedrock":
return httpx.Response(200, json={
"output": {"message": {"role": "assistant", "content": [{"text": content}]}},
"stopReason": "end_turn", "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15},
"metrics": {"latencyMs": 1},
})
return httpx.Response(200, json={
"id": "chatcmpl-audit", "object": "chat.completion", "created": 0, "model": "gpt-5.6",
"choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
})
async def capture(kwargs, response_obj, start_time, end_time):
logs.put_nowait(kwargs["standard_logging_object"])
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client:
handler: Final = AsyncHTTPHandler()
await handler.close()
handler.client = http_client
client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client) if provider == "openai" else handler
model: Final = {
"openai": "openai/gpt-5.6",
"anthropic": "anthropic/claude-haiku-4-5",
"bedrock": "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
}[provider]
async def run(marker: str) -> None:
await litellm.acompletion(
model=model, api_key="transport-only", client=client, max_tokens=128,
aws_access_key_id="transport-only", aws_secret_access_key="transport-only", aws_region_name="us-east-1",
messages=[{"role": "system", "content": "classifier-rubric"}, {"role": "user", "content": marker}],
metadata={"internal_call_origin": "autorouter_classifier"},
proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}},
success_callback=[capture], num_retries=0,
**({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}}
if provider == "openai" else {}),
)
await asyncio.gather(run("request-one"), run("request-two"))
requests: Final = await asyncio.wait_for(asyncio.gather(outbound.get(), outbound.get()), timeout=10)
payloads: Final = await asyncio.wait_for(asyncio.gather(logs.get(), logs.get()), timeout=10)
for payload in payloads:
snapshot: Final = payload["classifier_input"]
assert snapshot in requests
assert "source-only" not in json.dumps(snapshot)
assert "classifier-rubric" in json.dumps(snapshot)
assert "transport-only" not in json.dumps(snapshot)
assert "header-only-secret" not in json.dumps(snapshot)
assert "SIMPLE" in json.dumps(payload["response"])
marker: Final = "request-one" if "request-one" in json.dumps(snapshot) else "request-two"
assert payload["originating_request_masked"] == {"input": f"source-only-{marker}"}
assert classifier_input_snapshot(snapshot) is not None
if provider != "openai":
assert all("system" in request for request in requests)
@pytest.mark.parametrize("redaction", ["none", "global", "request", "header"])
@pytest.mark.parametrize("status", ["success", "failure"])
def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_obj, monkeypatch, redaction, status):
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
monkeypatch.setattr(litellm, "turn_off_message_logging", redaction == "global")
params: Final = {
"metadata": {"internal_call_origin": "autorouter_classifier", **(
{"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {}
)},
"proxy_server_request": {"body": {}, "originating_request_masked": {"input": "source-only"}},
}
logging_obj.model_call_details["litellm_params"] = params
logging_obj.model_call_details["standard_callback_dynamic_params"] = (
{"turn_off_message_logging": True} if redaction == "request" else {}
)
logging_obj.pre_call(
input=[], api_key=None, additional_args={"complete_input_dict": {"system": "rubric", "messages": []}}
)
now: Final = datetime.datetime.now()
payload: Final = get_standard_logging_object_payload(
kwargs={**logging_obj.model_call_details, "call_type": "completion"}, init_response_obj={},
start_time=now, end_time=now, logging_obj=logging_obj, status=status,
)
assert payload is not None
if redaction == "none":
assert payload["classifier_input"] == {"system": "rubric", "messages": []}
assert payload["originating_request_masked"] == {"input": "source-only"}
else:
assert "classifier_input" not in payload
assert "originating_request_masked" not in payload
@pytest.mark.parametrize("call_type,origin", [("completion", None), ("aembedding", "autorouter_classifier")])
def test_classifier_audit_is_not_added_to_other_calls(logging_obj, call_type, origin):
logging_obj.call_type = call_type
logging_obj.model_call_details["litellm_params"] = {"metadata": {"internal_call_origin": origin}}
logging_obj.pre_call(input=[], api_key=None, additional_args={"complete_input_dict": {"input": "embedding"}})
assert logging_obj.classifier_input is None

View file

@ -6,6 +6,7 @@ but litellm_params["litellm_metadata"] is None.
"""
import threading
from typing import Final
from types import SimpleNamespace
import pytest
@ -776,6 +777,29 @@ class TestPerformRedaction:
assert response_obj.choices[0].message.content == "secret content"
@pytest.mark.parametrize("callback_only", [False, True])
def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callback_only: bool) -> None:
audit: Final = {"classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}}
details: Final = {
"standard_logging_object": {**audit, "messages": [], "response": {}},
"litellm_params": {"proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]}},
}
logger: Final = CustomLogger()
logger.turn_off_message_logging = True
if callback_only:
redacted: Final = logger.redact_standard_logging_payload_from_model_call_details(details)
assert "classifier_input" not in redacted["standard_logging_object"]
assert "originating_request_masked" not in redacted["standard_logging_object"]
assert "originating_request_masked" not in redacted["litellm_params"]["proxy_server_request"]
assert details["standard_logging_object"]["classifier_input"] == audit["classifier_input"]
assert details["litellm_params"]["proxy_server_request"]["originating_request_masked"] == audit["originating_request_masked"]
else:
perform_redaction(details, result=None)
assert "classifier_input" not in details["standard_logging_object"]
assert "originating_request_masked" not in details["standard_logging_object"]
assert "originating_request_masked" not in details["litellm_params"]["proxy_server_request"]
def test_unredactable_result_is_not_deepcopied(self):
"""A result shape no branch can redact must not be deepcopied.

View file

@ -5703,6 +5703,29 @@ def _cold_storage_handler(payload):
return ColdStorageHandler(cold_storage_logger=logger), logger
@pytest.mark.asyncio
@pytest.mark.parametrize("cold_has_audit", [False, True])
async def test_resolve_payload_recovers_truncated_classifier_audit_without_losing_existing_fields(cold_has_audit):
full_audit = {"classifier_input": {"system": "full rubric"}, "originating_request_masked": {"input": "source"}}
truncated_request = {"model": "classifier", "classifier_input": {"system": "litellm_truncated"}}
handler, logger = _cold_storage_handler({
"proxy_server_request": {"body": {}}, **(full_audit if cold_has_audit else {}),
})
row = {
"messages": '[{"role":"user","content":"ask"}]', "response": '{"tier":"SIMPLE"}',
"proxy_server_request": json.dumps(truncated_request), "metadata": {"cold_storage_object_key": "k/audit.json"},
}
resolved = await spend_management_endpoints._resolve_request_response_payload(row, cold_storage_handler=handler)
assert logger.requested_object_keys == ["k/audit.json"]
assert resolved.messages == row["messages"]
assert resolved.response == row["response"]
if cold_has_audit:
assert resolved.proxy_server_request["classifier_input"] == full_audit["classifier_input"]
assert resolved.proxy_server_request["originating_request_masked"] == full_audit["originating_request_masked"]
else:
assert resolved.proxy_server_request == row["proxy_server_request"]
@pytest.mark.parametrize(
"value, expected",
[

View file

@ -67,6 +67,30 @@ def _get_additional_usage_values_for_usage(usage: litellm.Usage) -> dict:
return metadata["additional_usage_values"]
@pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)])
def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch, store_prompts, redact):
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": store_prompts})
audit: Final = {
"classifier_input": {"system": "rubric" * 1000, "messages": [{"role": "user", "content": "ask"}]},
"originating_request_masked": {"input": "source-only", "api_key": "REDACTED"},
}
stored: Final = json.loads(_get_proxy_server_request_for_spend_logs_payload(
metadata={}, litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}},
kwargs={"standard_logging_object": audit, "standard_callback_dynamic_params": {"turn_off_message_logging": redact}},
))
if not store_prompts or redact:
assert "classifier_input" not in stored
assert "originating_request_masked" not in stored
else:
assert stored["classifier_input"]["messages"] == audit["classifier_input"]["messages"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in json.dumps(stored["classifier_input"])
assert stored["originating_request_masked"]["input"] == "source-only"
assert stored["model"] == "classifier"
assert audit["classifier_input"]["system"] == "rubric" * 1000
def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_tokens():
additional_usage_values = _get_additional_usage_values_for_usage(
litellm.Usage(

View file

@ -2940,6 +2940,29 @@ class TestLLMClassifier:
"REASONING",
]
@pytest.mark.asyncio
@pytest.mark.parametrize("source_body", [
{"model": "router", "messages": [{"role": "user", "content": "source-only"}]},
{"model": "router", "system": "source-only", "messages": [{"role": "user", "content": "ask"}]},
{"model": "router", "instructions": "source-only", "input": "ask"},
])
async def test_classifier_source_is_masked_and_separate_from_provider_input(
self, llm_complexity_router, mock_router_instance, source_body
):
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
outcome = await llm_complexity_router.aclassify(
"classify-this-ask", request_kwargs={"proxy_server_request": {
"body": {**source_body, "metadata": {"authorization": "source-secret"}}
}}
)
assert outcome.cause == "llm_classifier"
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
source = call_kwargs["proxy_server_request"]["originating_request_masked"]
assert source == {**source_body, "metadata": {"authorization": "REDACTED"}}
assert "source-only" not in str(call_kwargs["messages"])
assert "source-only" not in str(call_kwargs["proxy_server_request"]["body"])
assert "classify-this-ask" in str(call_kwargs["messages"])
@pytest.mark.asyncio
@pytest.mark.parametrize("reasoning_effort", [None, "none", "low"], ids=["omitted", "none", "low"])
async def test_classifier_reasoning_effort_reaches_only_classifier_call(

View file

@ -0,0 +1,56 @@
import { render, screen, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi } from "vitest";
import { ClassifierAuditView } from "./ClassifierAuditView";
vi.mock("./JsonViewer", () => ({
JsonViewer: ({ data }: { data: unknown }) => <pre>{JSON.stringify(data)}</pre>,
}));
describe("ClassifierAuditView", () => {
it("separates and copies the provider input, source request, and returned verdict", async () => {
const user = userEvent.setup();
const input = { system: "classification rubric", messages: [{ role: "user", content: "classify this" }] };
render(
<ClassifierAuditView
request={{ classifier_input: input, originating_request_masked: { input: "source-only", api_key: "REDACTED" } }}
response={{ tier: "SIMPLE", reason: "a greeting" }}
/>,
);
const classifier = within(screen.getByRole("region", { name: "Classifier input" }));
expect(classifier.getByText(/classification rubric/)).toBeInTheDocument();
expect(classifier.queryByText(/source-only/)).not.toBeInTheDocument();
expect(
within(screen.getByRole("region", { name: "Originating request, credentials masked" })).getByText(/source-only/),
).toBeInTheDocument();
expect(
within(screen.getByRole("region", { name: "Classifier response" })).getByText(/a greeting/),
).toBeInTheDocument();
await user.click(classifier.getByRole("button", { name: "Copy Classifier input" }));
expect(await navigator.clipboard.readText()).toBe(JSON.stringify(input, null, 2));
});
it("does not present legacy source messages as captured classifier input", () => {
render(<ClassifierAuditView request={{ messages: [{ content: "legacy source" }] }} response={undefined} />);
expect(screen.getAllByText("Not captured or message logging disabled")).toHaveLength(3);
expect(screen.queryByRole("button", { name: "Copy Classifier input" })).not.toBeInTheDocument();
});
it("labels truncated input without marking a complete source request as truncated", () => {
render(
<ClassifierAuditView
request={{
classifier_input: { system: "partial rubric...litellm_truncated" },
originating_request_masked: { input: "source" },
}}
response={{ tier: "SIMPLE" }}
/>,
);
expect(within(screen.getByRole("region", { name: "Classifier input" })).getByRole("status")).toHaveTextContent(
"This stored copy is truncated",
);
expect(
within(screen.getByRole("region", { name: "Originating request, credentials masked" })).queryByRole("status"),
).not.toBeInTheDocument();
});
});

View file

@ -0,0 +1,52 @@
import CopyButton from "@/components/shared/CopyButton";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import type { ReactNode } from "react";
import { JsonViewer } from "./JsonViewer";
interface ClassifierAuditViewProps {
request: Record<string, unknown>;
response: unknown;
}
export function ClassifierAuditView({ request, response }: ClassifierAuditViewProps) {
return (
<div className="mb-6 space-y-4">
<AuditField title="Classifier input" value={request.classifier_input}>
Provider request payload. A cached call or disabled message logging may have no capture.
</AuditField>
<AuditField title="Originating request, credentials masked" value={request.originating_request_masked}>
Comparison only. This source request was not appended to the classifier input.
</AuditField>
<AuditField title="Classifier response" value={response}>
The returned verdict and any explanation supplied by the classifier. Later routing rules may change the tier.
</AuditField>
</div>
);
}
function AuditField({ title, value, children }: { title: string; value: unknown; children: ReactNode }) {
const serialized = JSON.stringify(value);
const truncated = serialized?.includes("litellm_truncated") ?? false;
return (
<Card size="sm" role="region" aria-label={title}>
<CardHeader>
<CardTitle>{title}</CardTitle>
{value != null && <CopyButton value={JSON.stringify(value, null, 2)} label={`Copy ${title}`} />}
</CardHeader>
<CardContent>
<p className="mb-3 text-sm text-muted-foreground">{children}</p>
{truncated && (
<p role="status" className="mb-3 text-sm text-warning">
This stored copy is truncated. The complete payload is unavailable from the configured log storage.
</p>
)}
{value == null ? (
<p className="text-sm text-muted-foreground">Not captured or message logging disabled</p>
) : (
<JsonViewer data={value} mode="formatted" />
)}
</CardContent>
</Card>
);
}

View file

@ -48,6 +48,8 @@ import {
} from "./constants";
import { ToolsSection } from "../ToolsSection";
import { PrettyMessagesView } from "./PrettyMessagesView";
import { ClassifierAuditView } from "./ClassifierAuditView";
import { AUTOROUTER_CLASSIFIER_ORIGIN } from "./ClassifyTag";
export interface LogDetailContentProps {
logEntry: LogEntry;
@ -68,6 +70,9 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok
const metadata = logEntry.metadata || {};
const hasError = metadata.status === "failure";
const errorInfo = hasError ? metadata.error_information : null;
const isClassifier =
metadata.internal_call_origin === AUTOROUTER_CLASSIFIER_ORIGIN &&
(logEntry.call_type === "completion" || logEntry.call_type === "acompletion");
const hasMessages = checkHasMessages(logEntry.messages);
const hasResponse = checkHasResponse(logEntry.response);
@ -196,7 +201,11 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok
Loading request &amp; response data...
</div>
</div>
) : (
) : null}
{!isLoadingDetails && isClassifier && (
<ClassifierAuditView request={getRawRequest()} response={getFormattedResponse()} />
)}
{!isLoadingDetails && !isClassifier && (
<RequestResponseSection
hasResponse={hasResponse}
hasError={hasError}
@ -541,22 +550,20 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata:
)}
<DescriptionItem label="Retries">
{metadata?.attempted_retries !== undefined && metadata?.attempted_retries !== null ? (
metadata.attempted_retries > 0 ? (
<>
{metadata.attempted_retries}
{metadata.max_retries !== undefined && metadata.max_retries !== null
? ` / ${metadata.max_retries}`
: ""}
</>
) : (
<Badge variant="secondary" className="bg-success/15 text-success">
None
</Badge>
)
) : (
"-"
{metadata?.attempted_retries != null && metadata.attempted_retries > 0 && (
<>
{metadata.attempted_retries}
{metadata.max_retries !== undefined && metadata.max_retries !== null
? ` / ${metadata.max_retries}`
: ""}
</>
)}
{metadata?.attempted_retries != null && metadata.attempted_retries <= 0 && (
<Badge variant="secondary" className="bg-success/15 text-success">
None
</Badge>
)}
{metadata?.attempted_retries == null && "-"}
</DescriptionItem>
<DescriptionItem label="Start Time">
@ -602,16 +609,10 @@ function RequestResponseSection({
const totalTokens = promptTokens + completionTokens;
const costBreakdown = logEntry.metadata?.cost_breakdown;
const useCostBreakdown = costBreakdown?.input_cost !== undefined && costBreakdown?.output_cost !== undefined;
const inputCost = useCostBreakdown
? costBreakdown!.input_cost ?? 0
: totalTokens > 0
? (totalSpend * promptTokens) / totalTokens
: 0;
const outputCost = useCostBreakdown
? costBreakdown!.output_cost ?? 0
: totalTokens > 0
? (totalSpend * completionTokens) / totalTokens
: 0;
const estimatedInputCost = totalTokens > 0 ? (totalSpend * promptTokens) / totalTokens : 0;
const estimatedOutputCost = totalTokens > 0 ? (totalSpend * completionTokens) / totalTokens : 0;
const inputCost = useCostBreakdown ? costBreakdown!.input_cost ?? 0 : estimatedInputCost;
const outputCost = useCostBreakdown ? costBreakdown!.output_cost ?? 0 : estimatedOutputCost;
return (
<div className="bg-card rounded-lg shadow-sm w-full max-w-full overflow-hidden mb-6">