diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 8f03e08f02d..473bf223105 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -927,6 +927,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac # Handle turn_off_message_logging - redact messages and responses (if not already excluded) if turn_off_message_logging: + from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit + + for field in CLASSIFIER_AUDIT_FIELDS: + standard_logging_object_copy.pop(field, None) + params: Final = model_call_details_copy.get("litellm_params") + request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None + if isinstance(params, dict) and isinstance(request, dict): + model_call_details_copy["litellm_params"] = { + **params, + "proxy_server_request": without_classifier_audit(request), + } redacted_str: Final = "redacted-by-litellm" if "messages" not in (excluded_fields or []) and standard_logging_object_copy.get("messages") is not None: diff --git a/litellm/litellm_core_utils/classifier_logging.py b/litellm/litellm_core_utils/classifier_logging.py new file mode 100644 index 00000000000..b9cb52a190d --- /dev/null +++ b/litellm/litellm_core_utils/classifier_logging.py @@ -0,0 +1,59 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue, TypeAdapter, ValidationError + +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ClassifierAudit + +CLASSIFIER_AUDIT_FIELDS: Final = ("classifier_input", "originating_request_masked") +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def classifier_input_snapshot(value: object, *, openai_sdk: bool = False) -> Mapping[str, JsonValue] | None: + if openai_sdk and isinstance(value, Mapping): + body: Final = { + key: item for key, item in value.items() if key not in ("extra_headers", "extra_query", "extra_body") + } + extra_body: Final = value.get("extra_body") + return classifier_input_snapshot({**body, **extra_body} if isinstance(extra_body, Mapping) else body) + try: + return ( + _JSON_OBJECT.validate_json(value) + if isinstance(value, (str, bytes)) + else _JSON_OBJECT.validate_python(value) + ) + except ValidationError: + return None + + +def is_classifier_call(call_type: str, params: Mapping[str, object]) -> bool: + return call_type in ("completion", "acompletion") and any( + isinstance(metadata := params.get(key), Mapping) + and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == AUTOROUTER_CLASSIFIER_CALL_ORIGIN + for key in ("metadata", "litellm_metadata") + ) + + +def masked_originating_request(request_kwargs: Mapping[str, object] | None) -> Mapping[str, JsonValue] | None: + request: Final = (request_kwargs or {}).get("proxy_server_request") + body: Final = request.get("body") if isinstance(request, Mapping) else None + if not isinstance(body, Mapping): + return None + serializable: Final = classifier_input_snapshot(safe_dumps(body)) + return classifier_input_snapshot(redact_credentials_in_payload(serializable)) if serializable is not None else None + + +def classifier_audit_fields(payload: Mapping[str, object]) -> ClassifierAudit: + classifier_input: Final = classifier_input_snapshot(payload.get("classifier_input")) + originating_request: Final = classifier_input_snapshot(payload.get("originating_request_masked")) + return { + **(ClassifierAudit(classifier_input=classifier_input) if classifier_input is not None else {}), + **(ClassifierAudit(originating_request_masked=originating_request) if originating_request is not None else {}), + } + + +def without_classifier_audit(payload: Mapping[str, object]) -> dict[str, object]: + return {key: value for key, value in payload.items() if key not in CLASSIFIER_AUDIT_FIELDS} diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cb9209be267..f85aa32829f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -17,7 +17,7 @@ from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast from httpx import Response -from pydantic import BaseModel +from pydantic import BaseModel, JsonValue import litellm from litellm import ( @@ -64,6 +64,11 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger +from litellm.litellm_core_utils.classifier_logging import ( + classifier_audit_fields, + classifier_input_snapshot, + is_classifier_call, +) from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.internal_call_metadata import ( @@ -89,6 +94,7 @@ from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, redact_streaming_responses_for_custom_logger, + should_redact_message_logging, ) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse @@ -473,6 +479,7 @@ class Logging(LiteLLMLoggingBaseClass): stream_options = None litellm_request_debug: bool = False streamed_anthropic_message_id: str | None = None + classifier_input: Mapping[str, JsonValue] | None = None def __init__( self, @@ -1211,6 +1218,14 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["api_key"] = api_key self.model_call_details["additional_args"] = additional_args self.model_call_details["log_event_type"] = "pre_api_call" + if is_classifier_call(self.call_type, self.model_call_details.get("litellm_params") or {}): + self.classifier_input = ( + None + if should_redact_message_logging(self.model_call_details) + else classifier_input_snapshot( + additional_args.get("complete_input_dict"), openai_sdk=additional_args.get("openai_sdk") is True + ) + ) if model: # if model name was changes pre-call, overwrite the initial model call name with the new one self.model_call_details["model"] = model self.model_call_details["litellm_params"]["api_base"] = self._get_masked_api_base( @@ -6293,6 +6308,16 @@ def get_standard_logging_object_payload( ) payload: Final[StandardLoggingPayload] = StandardLoggingPayload( + **( + classifier_audit_fields( + { + "classifier_input": logging_obj.classifier_input, + "originating_request_masked": proxy_server_request.get("originating_request_masked"), + } + ) + if is_classifier_call(call_type or "", litellm_params) and not should_redact_message_logging(kwargs) + else {} + ), id=str(id), litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), trace_id=StandardLoggingPayloadSetup.get_standard_logging_payload_trace_id( diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index a567854d767..c97566c21a8 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any, Final import litellm from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) @@ -183,6 +184,9 @@ def _redact_standard_logging_object(model_call_details: dict): redacted_str: Final = REDACTED_BY_LITELLM + for field in CLASSIFIER_AUDIT_FIELDS: + standard_logging_object.pop(field, None) + if standard_logging_object.get("messages") is not None: standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}] @@ -254,6 +258,12 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons copy via redact_streaming_responses_for_custom_logger instead. """ # Redact model_call_details + for field in CLASSIFIER_AUDIT_FIELDS: + model_call_details.pop(field, None) + params: Final = model_call_details.get("litellm_params") + request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None + if isinstance(params, dict) and isinstance(request, Mapping): + model_call_details["litellm_params"] = {**params, "proxy_server_request": without_classifier_audit(request)} model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] model_call_details["prompt"] = "" model_call_details["input"] = "" diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index edc8d64d9c2..5f04ebe0c01 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -797,6 +797,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): "api_base": openai_client._base_url._uri_reference, "acompletion": acompletion, "complete_input_dict": data, + "openai_sdk": True, }, ) @@ -938,6 +939,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): "api_base": openai_aclient._base_url._uri_reference, "acompletion": True, "complete_input_dict": data, + "openai_sdk": True, }, ) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index f8831ca4152..ee43972309b 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -26,7 +26,8 @@ from typing_extensions import ReadOnly import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME +from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME +from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields, classifier_input_snapshot from litellm.proxy._types import * from litellm.proxy._types import ProviderBudgetResponse, ProviderBudgetResponseObject from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -3099,7 +3100,11 @@ async def _resolve_request_response_payload( proxy_server_request: Final = row.get("proxy_server_request") pg_payload: Final = RequestResponsePayload(messages, response, proxy_server_request) - if ( + stored_request: Final = classifier_input_snapshot(proxy_server_request) + truncated_audit: Final = bool(stored_request and classifier_audit_fields(stored_request)) and ( + LITELLM_TRUNCATED_PAYLOAD_FIELD in str(proxy_server_request) + ) + if not truncated_audit and ( _spend_log_field_has_content(messages) or _spend_log_field_has_content(response) or _spend_log_field_has_content(proxy_server_request) @@ -3124,10 +3129,19 @@ async def _resolve_request_response_payload( if payload is None: return pg_payload + cold_audit: Final = classifier_audit_fields(payload) + resolved_request: Final = ( + {**(classifier_input_snapshot(payload.get("proxy_server_request")) or stored_request or {}), **cold_audit} + if cold_audit + else payload.get("proxy_server_request") + ) + if truncated_audit: + return RequestResponsePayload(messages, response, resolved_request if cold_audit else proxy_server_request) + return RequestResponsePayload( messages=payload.get("messages"), response=payload.get("response"), - proxy_server_request=payload.get("proxy_server_request"), + proxy_server_request=resolved_request, ) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index a21d761996f..da55d1cb153 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -22,6 +22,7 @@ from litellm.constants import ( from litellm.constants import ( MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, ) +from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, @@ -1251,6 +1252,10 @@ def _get_proxy_server_request_for_spend_logs_payload( if _proxy_server_request is not None: _request_body = _proxy_server_request.get("body", {}) or {} + standard_payload: Final = (kwargs or {}).get("standard_logging_object") + if isinstance(standard_payload, Mapping): + _request_body = {**_request_body, **classifier_audit_fields(standard_payload)} + if kwargs is not None: realtime_tools: Final = kwargs.get("realtime_tools") if realtime_tools: diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index faafcea404a..162fcf2efb0 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -36,6 +36,7 @@ from litellm.constants import ( SESSION_ID_GENERATED_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.classifier_logging import masked_originating_request from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -1924,11 +1925,8 @@ class ComplexityRouter(CustomLogger): Call the configured classifier model with a system/user role split and prior-turn context. Builds a structured classification prompt with: - - System message: the stable classifier rubric AND the caller's own system prompt (task - constraints). This is the largest, most repeated part of the call, so keeping it in the - system role lets the provider prompt-cache it across a session's classifier calls. - - User message: the variable payload -- a few prior user turns for context and the current - ask to classify. + - System message: the stable classifier rubric. + - User message: the caller's system prompt quoted as task context, prior turns, and the current ask. Args: prompt: The current user ask text (already extracted as the real human ask, not tool results) @@ -2005,12 +2003,13 @@ class ComplexityRouter(CustomLogger): classifier_call_params = MappingProxyType({"reasoning_effort": llm_config.reasoning_effort}) proxy_server_request: Final = { + "originating_request_masked": masked_originating_request(request_kwargs), "body": { "model": llm_config.model, "messages": messages_for_call, "response_format": response_format, **classifier_call_params, - } + }, } classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000 diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ab0cc5f959c..58b940227f8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -35,6 +35,7 @@ from pydantic import ( BaseModel, ConfigDict, Field, + JsonValue, PrivateAttr, SkipValidation, field_serializer, @@ -3373,7 +3374,12 @@ class StandardAuditLogPayload(TypedDict): updated_values: str | None -class StandardLoggingPayload(TypedDict): +class ClassifierAudit(TypedDict, total=False): + classifier_input: ReadOnly[Mapping[str, JsonValue]] + originating_request_masked: ReadOnly[Mapping[str, JsonValue]] + + +class StandardLoggingPayload(ClassifierAudit): id: str trace_id: str # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries) session_id: str # End-user/conversation session id (litellm_session_id), independent of trace_id diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 6b69677d490..4d40f19e67b 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -1,4 +1,5 @@ # Reliability & Performance (behavior features). Grounded in litellm/router.py + router_strategy/ + router_utils/. +- {id: reliability.routing.classifier_audit.separates_provider_input_and_source, module: reliability, tier: P1, behavior: routing, variant: classifier_audit, assertions: [separates_provider_input_and_source], exercised_on: [chat_completions, messages, responses], source: "litellm/litellm_core_utils/classifier_logging.py", rationale: "Classifier spend details distinguish provider input from the credential-masked originating request across all three request surfaces"} - {id: reliability.fallback.5xx.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: "5xx", assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:2024", rationale: "Reroute on provider 5xx to alternate deployment"} - {id: reliability.fallback.context_window.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: context_window, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6108", rationale: "Fallback when model exceeds context limit"} - {id: reliability.fallback.content_policy.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: content_policy, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6023", rationale: "Reroute on content-policy violation"} diff --git a/tests/e2e/router/test_classifier_audit_e2e.py b/tests/e2e/router/test_classifier_audit_e2e.py new file mode 100644 index 00000000000..6b444441477 --- /dev/null +++ b/tests/e2e/router/test_classifier_audit_e2e.py @@ -0,0 +1,143 @@ +import json +import os +from collections.abc import Iterator +from contextlib import ExitStack +from dataclasses import dataclass +from typing import Final, Literal + +import pytest +from pydantic import BaseModel, Field, JsonValue, TypeAdapter + +from e2e_config import unique_marker +from e2e_http import AnthropicHeaders, NoBody, unwrap +from models import ChatMessage, KeyGenerateBody, LiteLLMParamsBody +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + + +class AuditMetadata(BaseModel): + source_marker: str + authorization: str = "synthetic-audit-secret" + + +class AuditHeaders(AnthropicHeaders): + enable_redaction: str | None = Field(default=None, serialization_alias="x-litellm-enable-message-redaction") + + +class AuditRequest(BaseModel): + model: str + messages: list[ChatMessage] | None = None + system: str | None = None + instructions: str | None = None + input: str | None = None + max_tokens: int | None = None + max_output_tokens: int | None = None + metadata: AuditMetadata | None = None + litellm_metadata: AuditMetadata | None = None + + +class AuditResponse(BaseModel): + id: str + + +class AuditDetail(BaseModel): + proxy_server_request: dict[str, JsonValue] | str | None = None + response: dict[str, JsonValue] | str | None = None + + +@dataclass(frozen=True, slots=True) +class AuditDeployment: + alias: str + key: str + + +@pytest.fixture +def audit_deployment(proxy: ProxyClient, provider: str) -> Iterator[AuditDeployment]: + marker: Final = unique_marker() + classifier: Final = f"audit-classifier-{marker}" + target: Final = f"audit-target-{marker}" + alias: Final = f"audit-router-{marker}" + model: Final = os.environ.get( + f"E2E_CHEAP_{provider.upper()}_MODEL", "gpt-5.6" if provider == "openai" else "claude-haiku-4-5" + ) + params: Final = LiteLLMParamsBody( + model=f"{provider}/{model}", + api_key=os.environ.get(f"{provider.upper()}_API_KEY") or f"os.environ/{provider.upper()}_API_KEY", + api_base=os.environ.get(f"{provider.upper()}_API_BASE"), + ) + with ExitStack() as stack: + for name in (classifier, target): + stack.callback(proxy.delete_model, proxy.create_model(name, params)) + stack.callback(proxy.delete_model, proxy.create_model(alias, LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config={ + "classifier_type": "llm", + "classifier_llm_config": {"model": classifier, "timeout_ms": 30000}, + "tiers": {tier: target for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")}, + }, + ))) + key: Final = proxy.generate_key(KeyGenerateBody(models=[alias, classifier, target])) + stack.callback(proxy.delete_key, key) + yield AuditDeployment(alias, key) + + +class TestClassifierAudit: + @pytest.mark.covers( + "reliability.routing.classifier_audit.separates_provider_input_and_source", + exercised_on=("chat_completions", "messages", "responses"), + ) + @pytest.mark.parametrize("surface", ["chat_completions", "messages", "responses"]) + @pytest.mark.parametrize("provider", ["openai", "anthropic"]) + @pytest.mark.parametrize("redact", [False, True]) + def test_classifier_audit_separates_input_and_masked_source( + self, proxy: ProxyClient, audit_deployment: AuditDeployment, surface: Literal["chat_completions", "messages", "responses"], + redact: bool, + ) -> None: + marker: Final = unique_marker() + source_marker: Final = f"source-only-{marker}" + prompt: Final = f"Reply with hello. Request label {marker}" + metadata: Final = AuditMetadata(source_marker=source_marker) + body: Final = AuditRequest( + model=audit_deployment.alias, + messages=[ChatMessage(role="user", content=prompt)] if surface != "responses" else None, + input=prompt if surface == "responses" else None, + system="Be concise" if surface == "messages" else None, + instructions="Be concise" if surface == "responses" else None, + max_tokens=128 if surface != "responses" else None, + max_output_tokens=128 if surface == "responses" else None, + metadata=metadata if surface != "responses" else None, + litellm_metadata=metadata if surface == "responses" else None, + ) + path: Final = {"chat_completions": "/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}[surface] + response: Final = unwrap(proxy.transport.post( + path, headers=AuditHeaders(authorization=f"Bearer {audit_deployment.key}", enable_redaction="true" if redact else None), + json=body, response_type=AuditResponse, + )) + assert response.id + rows: Final = proxy.poll_logs_for_key(audit_deployment.key, min_rows=2) + assert len(rows) == 2, "Expected a classifier spend row and a routed response spend row" + details: Final = tuple( + unwrap(proxy.transport.get( + f"/spend/logs/ui/{row.request_id}", headers=proxy.transport.master, + params=NoBody(), response_type=AuditDetail, + )) for row in rows + ) + adapter: Final = TypeAdapter(dict[str, JsonValue]) + requests: Final = tuple( + adapter.validate_json(detail.proxy_server_request) if isinstance(detail.proxy_server_request, str) + else detail.proxy_server_request or {} for detail in details + ) + audits: Final = tuple(item for item in requests if "classifier_input" in item) + if redact: + assert audits == () + assert all("originating_request_masked" not in item for item in requests) + return + assert len(audits) == 1, "The audit belongs only to the classifier call" + audit: Final = audits[0] + assert marker in json.dumps(audit["classifier_input"]) + assert source_marker not in json.dumps(audit["classifier_input"]) + assert source_marker in json.dumps(audit["originating_request_masked"]) + assert "synthetic-audit-secret" not in json.dumps(audit) + assert audit_deployment.alias in json.dumps(audit["originating_request_masked"]) + assert any("tier" in str(detail.response) for detail in details) diff --git a/tests/test_litellm/litellm_core_utils/test_classifier_logging.py b/tests/test_litellm/litellm_core_utils/test_classifier_logging.py new file mode 100644 index 00000000000..c923225c750 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_classifier_logging.py @@ -0,0 +1,38 @@ +from typing import Final + +import pytest + +from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot, masked_originating_request + + +@pytest.mark.parametrize("encoded", [False, True]) +def test_classifier_snapshot_preserves_provider_shape_and_is_independent(encoded: bool) -> None: + import json + + provider_body: Final = {"system": [{"text": "rubric"}], "messages": [{"role": "user", "content": "ask"}]} + snapshot: Final = classifier_input_snapshot(json.dumps(provider_body) if encoded else provider_body) + assert snapshot == provider_body + provider_body["messages"][0]["content"] = "later mutation" + assert snapshot == {"system": [{"text": "rubric"}], "messages": [{"role": "user", "content": "ask"}]} + + +def test_originating_snapshot_masks_nested_credentials_without_altering_source() -> None: + body: Final = { + "model": "router", + "input": [{"type": "message", "role": "user", "content": "source-only"}], + "api_key": "short", + "metadata": {"nested": [{"Authorization": "Bearer secret", "access_token": 123}]}, + } + snapshot: Final = masked_originating_request({"proxy_server_request": {"body": body}}) + assert snapshot is not None + assert snapshot["model"] == "router" + assert snapshot["input"] == body["input"] + assert snapshot["api_key"] == "REDACTED" + assert snapshot["metadata"] == {"nested": [{"Authorization": "REDACTED", "access_token": "REDACTED"}]} + assert body["api_key"] == "short" + assert body["metadata"]["nested"][0]["Authorization"] == "Bearer secret" + + +@pytest.mark.parametrize("value", [None, "not-json", [], {"messages": object()}]) +def test_invalid_provider_payload_is_not_reported_as_captured(value: object) -> None: + assert classifier_input_snapshot(value) is None diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 6aa77745e3d..28545bb2349 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3,7 +3,7 @@ import contextlib import datetime import os import sys -from typing import Literal +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -6801,3 +6801,122 @@ def test_get_error_information_redacts_provider_key_from_upstream_url(): assert "REDACTED" in result["traceback"] assert "REDACTED" in result["error_message"] assert result["error_code"] == "400" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ["openai", "anthropic", "bedrock"]) +async def test_classifier_audit_matches_provider_transport(provider: str) -> None: + import json + + from openai import AsyncOpenAI + + from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + outbound: Final = asyncio.Queue() + logs: Final = asyncio.Queue() + + def respond(request: httpx.Request) -> httpx.Response: + outbound.put_nowait(json.loads(request.content)) + content: Final = '{"tier":"SIMPLE"}' + if provider == "anthropic": + return httpx.Response(200, json={ + "id": "msg-audit", "type": "message", "role": "assistant", "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": content}], "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }) + if provider == "bedrock": + return httpx.Response(200, json={ + "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, + "stopReason": "end_turn", "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + }) + return httpx.Response(200, json={ + "id": "chatcmpl-audit", "object": "chat.completion", "created": 0, "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }) + + async def capture(kwargs, response_obj, start_time, end_time): + logs.put_nowait(kwargs["standard_logging_object"]) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler: Final = AsyncHTTPHandler() + await handler.close() + handler.client = http_client + client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client) if provider == "openai" else handler + model: Final = { + "openai": "openai/gpt-5.6", + "anthropic": "anthropic/claude-haiku-4-5", + "bedrock": "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", + }[provider] + + async def run(marker: str) -> None: + await litellm.acompletion( + model=model, api_key="transport-only", client=client, max_tokens=128, + aws_access_key_id="transport-only", aws_secret_access_key="transport-only", aws_region_name="us-east-1", + messages=[{"role": "system", "content": "classifier-rubric"}, {"role": "user", "content": marker}], + metadata={"internal_call_origin": "autorouter_classifier"}, + proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, + success_callback=[capture], num_retries=0, + **({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}} + if provider == "openai" else {}), + ) + + await asyncio.gather(run("request-one"), run("request-two")) + requests: Final = await asyncio.wait_for(asyncio.gather(outbound.get(), outbound.get()), timeout=10) + payloads: Final = await asyncio.wait_for(asyncio.gather(logs.get(), logs.get()), timeout=10) + for payload in payloads: + snapshot: Final = payload["classifier_input"] + assert snapshot in requests + assert "source-only" not in json.dumps(snapshot) + assert "classifier-rubric" in json.dumps(snapshot) + assert "transport-only" not in json.dumps(snapshot) + assert "header-only-secret" not in json.dumps(snapshot) + assert "SIMPLE" in json.dumps(payload["response"]) + marker: Final = "request-one" if "request-one" in json.dumps(snapshot) else "request-two" + assert payload["originating_request_masked"] == {"input": f"source-only-{marker}"} + assert classifier_input_snapshot(snapshot) is not None + if provider != "openai": + assert all("system" in request for request in requests) + + +@pytest.mark.parametrize("redaction", ["none", "global", "request", "header"]) +@pytest.mark.parametrize("status", ["success", "failure"]) +def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_obj, monkeypatch, redaction, status): + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + monkeypatch.setattr(litellm, "turn_off_message_logging", redaction == "global") + params: Final = { + "metadata": {"internal_call_origin": "autorouter_classifier", **( + {"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {} + )}, + "proxy_server_request": {"body": {}, "originating_request_masked": {"input": "source-only"}}, + } + logging_obj.model_call_details["litellm_params"] = params + logging_obj.model_call_details["standard_callback_dynamic_params"] = ( + {"turn_off_message_logging": True} if redaction == "request" else {} + ) + logging_obj.pre_call( + input=[], api_key=None, additional_args={"complete_input_dict": {"system": "rubric", "messages": []}} + ) + now: Final = datetime.datetime.now() + payload: Final = get_standard_logging_object_payload( + kwargs={**logging_obj.model_call_details, "call_type": "completion"}, init_response_obj={}, + start_time=now, end_time=now, logging_obj=logging_obj, status=status, + ) + assert payload is not None + if redaction == "none": + assert payload["classifier_input"] == {"system": "rubric", "messages": []} + assert payload["originating_request_masked"] == {"input": "source-only"} + else: + assert "classifier_input" not in payload + assert "originating_request_masked" not in payload + + +@pytest.mark.parametrize("call_type,origin", [("completion", None), ("aembedding", "autorouter_classifier")]) +def test_classifier_audit_is_not_added_to_other_calls(logging_obj, call_type, origin): + logging_obj.call_type = call_type + logging_obj.model_call_details["litellm_params"] = {"metadata": {"internal_call_origin": origin}} + logging_obj.pre_call(input=[], api_key=None, additional_args={"complete_input_dict": {"input": "embedding"}}) + assert logging_obj.classifier_input is None diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 3be0bae4120..1ba9901aab0 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -6,6 +6,7 @@ but litellm_params["litellm_metadata"] is None. """ import threading +from typing import Final from types import SimpleNamespace import pytest @@ -776,6 +777,29 @@ class TestPerformRedaction: assert response_obj.choices[0].message.content == "secret content" + +@pytest.mark.parametrize("callback_only", [False, True]) +def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callback_only: bool) -> None: + audit: Final = {"classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}} + details: Final = { + "standard_logging_object": {**audit, "messages": [], "response": {}}, + "litellm_params": {"proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]}}, + } + logger: Final = CustomLogger() + logger.turn_off_message_logging = True + if callback_only: + redacted: Final = logger.redact_standard_logging_payload_from_model_call_details(details) + assert "classifier_input" not in redacted["standard_logging_object"] + assert "originating_request_masked" not in redacted["standard_logging_object"] + assert "originating_request_masked" not in redacted["litellm_params"]["proxy_server_request"] + assert details["standard_logging_object"]["classifier_input"] == audit["classifier_input"] + assert details["litellm_params"]["proxy_server_request"]["originating_request_masked"] == audit["originating_request_masked"] + else: + perform_redaction(details, result=None) + assert "classifier_input" not in details["standard_logging_object"] + assert "originating_request_masked" not in details["standard_logging_object"] + assert "originating_request_masked" not in details["litellm_params"]["proxy_server_request"] + def test_unredactable_result_is_not_deepcopied(self): """A result shape no branch can redact must not be deepcopied. diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 329a33eb440..a65209b1456 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -5703,6 +5703,29 @@ def _cold_storage_handler(payload): return ColdStorageHandler(cold_storage_logger=logger), logger +@pytest.mark.asyncio +@pytest.mark.parametrize("cold_has_audit", [False, True]) +async def test_resolve_payload_recovers_truncated_classifier_audit_without_losing_existing_fields(cold_has_audit): + full_audit = {"classifier_input": {"system": "full rubric"}, "originating_request_masked": {"input": "source"}} + truncated_request = {"model": "classifier", "classifier_input": {"system": "litellm_truncated"}} + handler, logger = _cold_storage_handler({ + "proxy_server_request": {"body": {}}, **(full_audit if cold_has_audit else {}), + }) + row = { + "messages": '[{"role":"user","content":"ask"}]', "response": '{"tier":"SIMPLE"}', + "proxy_server_request": json.dumps(truncated_request), "metadata": {"cold_storage_object_key": "k/audit.json"}, + } + resolved = await spend_management_endpoints._resolve_request_response_payload(row, cold_storage_handler=handler) + assert logger.requested_object_keys == ["k/audit.json"] + assert resolved.messages == row["messages"] + assert resolved.response == row["response"] + if cold_has_audit: + assert resolved.proxy_server_request["classifier_input"] == full_audit["classifier_input"] + assert resolved.proxy_server_request["originating_request_masked"] == full_audit["originating_request_masked"] + else: + assert resolved.proxy_server_request == row["proxy_server_request"] + + @pytest.mark.parametrize( "value, expected", [ diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 5a79560b972..adc15870968 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -67,6 +67,30 @@ def _get_additional_usage_values_for_usage(usage: litellm.Usage) -> dict: return metadata["additional_usage_values"] +@pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)]) +def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch, store_prompts, redact): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": store_prompts}) + audit: Final = { + "classifier_input": {"system": "rubric" * 1000, "messages": [{"role": "user", "content": "ask"}]}, + "originating_request_masked": {"input": "source-only", "api_key": "REDACTED"}, + } + stored: Final = json.loads(_get_proxy_server_request_for_spend_logs_payload( + metadata={}, litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}}, + kwargs={"standard_logging_object": audit, "standard_callback_dynamic_params": {"turn_off_message_logging": redact}}, + )) + if not store_prompts or redact: + assert "classifier_input" not in stored + assert "originating_request_masked" not in stored + else: + assert stored["classifier_input"]["messages"] == audit["classifier_input"]["messages"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in json.dumps(stored["classifier_input"]) + assert stored["originating_request_masked"]["input"] == "source-only" + assert stored["model"] == "classifier" + assert audit["classifier_input"]["system"] == "rubric" * 1000 + + def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_tokens(): additional_usage_values = _get_additional_usage_values_for_usage( litellm.Usage( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 51103297c58..45b2faea126 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2940,6 +2940,29 @@ class TestLLMClassifier: "REASONING", ] + @pytest.mark.asyncio + @pytest.mark.parametrize("source_body", [ + {"model": "router", "messages": [{"role": "user", "content": "source-only"}]}, + {"model": "router", "system": "source-only", "messages": [{"role": "user", "content": "ask"}]}, + {"model": "router", "instructions": "source-only", "input": "ask"}, + ]) + async def test_classifier_source_is_masked_and_separate_from_provider_input( + self, llm_complexity_router, mock_router_instance, source_body + ): + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')) + outcome = await llm_complexity_router.aclassify( + "classify-this-ask", request_kwargs={"proxy_server_request": { + "body": {**source_body, "metadata": {"authorization": "source-secret"}} + }} + ) + assert outcome.cause == "llm_classifier" + call_kwargs = mock_router_instance.acompletion.call_args.kwargs + source = call_kwargs["proxy_server_request"]["originating_request_masked"] + assert source == {**source_body, "metadata": {"authorization": "REDACTED"}} + assert "source-only" not in str(call_kwargs["messages"]) + assert "source-only" not in str(call_kwargs["proxy_server_request"]["body"]) + assert "classify-this-ask" in str(call_kwargs["messages"]) + @pytest.mark.asyncio @pytest.mark.parametrize("reasoning_effort", [None, "none", "low"], ids=["omitted", "none", "low"]) async def test_classifier_reasoning_effort_reaches_only_classifier_call( diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx new file mode 100644 index 00000000000..86e98de5557 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx @@ -0,0 +1,56 @@ +import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import { ClassifierAuditView } from "./ClassifierAuditView"; + +vi.mock("./JsonViewer", () => ({ + JsonViewer: ({ data }: { data: unknown }) => 
{JSON.stringify(data)}
, +})); + +describe("ClassifierAuditView", () => { + it("separates and copies the provider input, source request, and returned verdict", async () => { + const user = userEvent.setup(); + const input = { system: "classification rubric", messages: [{ role: "user", content: "classify this" }] }; + render( + , + ); + const classifier = within(screen.getByRole("region", { name: "Classifier input" })); + expect(classifier.getByText(/classification rubric/)).toBeInTheDocument(); + expect(classifier.queryByText(/source-only/)).not.toBeInTheDocument(); + expect( + within(screen.getByRole("region", { name: "Originating request, credentials masked" })).getByText(/source-only/), + ).toBeInTheDocument(); + expect( + within(screen.getByRole("region", { name: "Classifier response" })).getByText(/a greeting/), + ).toBeInTheDocument(); + await user.click(classifier.getByRole("button", { name: "Copy Classifier input" })); + expect(await navigator.clipboard.readText()).toBe(JSON.stringify(input, null, 2)); + }); + + it("does not present legacy source messages as captured classifier input", () => { + render(); + expect(screen.getAllByText("Not captured or message logging disabled")).toHaveLength(3); + expect(screen.queryByRole("button", { name: "Copy Classifier input" })).not.toBeInTheDocument(); + }); + + it("labels truncated input without marking a complete source request as truncated", () => { + render( + , + ); + expect(within(screen.getByRole("region", { name: "Classifier input" })).getByRole("status")).toHaveTextContent( + "This stored copy is truncated", + ); + expect( + within(screen.getByRole("region", { name: "Originating request, credentials masked" })).queryByRole("status"), + ).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx new file mode 100644 index 00000000000..913700efa9b --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx @@ -0,0 +1,52 @@ +import CopyButton from "@/components/shared/CopyButton"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { ReactNode } from "react"; +import { JsonViewer } from "./JsonViewer"; + +interface ClassifierAuditViewProps { + request: Record; + response: unknown; +} + +export function ClassifierAuditView({ request, response }: ClassifierAuditViewProps) { + return ( +
+ + Provider request payload. A cached call or disabled message logging may have no capture. + + + Comparison only. This source request was not appended to the classifier input. + + + The returned verdict and any explanation supplied by the classifier. Later routing rules may change the tier. + +
+ ); +} + +function AuditField({ title, value, children }: { title: string; value: unknown; children: ReactNode }) { + const serialized = JSON.stringify(value); + const truncated = serialized?.includes("litellm_truncated") ?? false; + + return ( + + + {title} + {value != null && } + + +

{children}

+ {truncated && ( +

+ This stored copy is truncated. The complete payload is unavailable from the configured log storage. +

+ )} + {value == null ? ( +

Not captured or message logging disabled

+ ) : ( + + )} + + + ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index 186cb66592b..e29f2810926 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -48,6 +48,8 @@ import { } from "./constants"; import { ToolsSection } from "../ToolsSection"; import { PrettyMessagesView } from "./PrettyMessagesView"; +import { ClassifierAuditView } from "./ClassifierAuditView"; +import { AUTOROUTER_CLASSIFIER_ORIGIN } from "./ClassifyTag"; export interface LogDetailContentProps { logEntry: LogEntry; @@ -68,6 +70,9 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok const metadata = logEntry.metadata || {}; const hasError = metadata.status === "failure"; const errorInfo = hasError ? metadata.error_information : null; + const isClassifier = + metadata.internal_call_origin === AUTOROUTER_CLASSIFIER_ORIGIN && + (logEntry.call_type === "completion" || logEntry.call_type === "acompletion"); const hasMessages = checkHasMessages(logEntry.messages); const hasResponse = checkHasResponse(logEntry.response); @@ -196,7 +201,11 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok Loading request & response data... - ) : ( + ) : null} + {!isLoadingDetails && isClassifier && ( + + )} + {!isLoadingDetails && !isClassifier && ( - {metadata?.attempted_retries !== undefined && metadata?.attempted_retries !== null ? ( - metadata.attempted_retries > 0 ? ( - <> - {metadata.attempted_retries} - {metadata.max_retries !== undefined && metadata.max_retries !== null - ? ` / ${metadata.max_retries}` - : ""} - - ) : ( - - None - - ) - ) : ( - "-" + {metadata?.attempted_retries != null && metadata.attempted_retries > 0 && ( + <> + {metadata.attempted_retries} + {metadata.max_retries !== undefined && metadata.max_retries !== null + ? ` / ${metadata.max_retries}` + : ""} + )} + {metadata?.attempted_retries != null && metadata.attempted_retries <= 0 && ( + + None + + )} + {metadata?.attempted_retries == null && "-"} @@ -602,16 +609,10 @@ function RequestResponseSection({ const totalTokens = promptTokens + completionTokens; const costBreakdown = logEntry.metadata?.cost_breakdown; const useCostBreakdown = costBreakdown?.input_cost !== undefined && costBreakdown?.output_cost !== undefined; - const inputCost = useCostBreakdown - ? costBreakdown!.input_cost ?? 0 - : totalTokens > 0 - ? (totalSpend * promptTokens) / totalTokens - : 0; - const outputCost = useCostBreakdown - ? costBreakdown!.output_cost ?? 0 - : totalTokens > 0 - ? (totalSpend * completionTokens) / totalTokens - : 0; + const estimatedInputCost = totalTokens > 0 ? (totalSpend * promptTokens) / totalTokens : 0; + const estimatedOutputCost = totalTokens > 0 ? (totalSpend * completionTokens) / totalTokens : 0; + const inputCost = useCostBreakdown ? costBreakdown!.input_cost ?? 0 : estimatedInputCost; + const outputCost = useCostBreakdown ? costBreakdown!.output_cost ?? 0 : estimatedOutputCost; return (