mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
feat(router): log classifier input and masked source request
This commit is contained in:
parent
6c69dd0f72
commit
153c962e15
20 changed files with 673 additions and 38 deletions
|
|
@ -927,6 +927,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
|
||||
# Handle turn_off_message_logging - redact messages and responses (if not already excluded)
|
||||
if turn_off_message_logging:
|
||||
from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit
|
||||
|
||||
for field in CLASSIFIER_AUDIT_FIELDS:
|
||||
standard_logging_object_copy.pop(field, None)
|
||||
params: Final = model_call_details_copy.get("litellm_params")
|
||||
request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None
|
||||
if isinstance(params, dict) and isinstance(request, dict):
|
||||
model_call_details_copy["litellm_params"] = {
|
||||
**params,
|
||||
"proxy_server_request": without_classifier_audit(request),
|
||||
}
|
||||
redacted_str: Final = "redacted-by-litellm"
|
||||
|
||||
if "messages" not in (excluded_fields or []) and standard_logging_object_copy.get("messages") is not None:
|
||||
|
|
|
|||
59
litellm/litellm_core_utils/classifier_logging.py
Normal file
59
litellm/litellm_core_utils/classifier_logging.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
|
||||
from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ClassifierAudit
|
||||
|
||||
CLASSIFIER_AUDIT_FIELDS: Final = ("classifier_input", "originating_request_masked")
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def classifier_input_snapshot(value: object, *, openai_sdk: bool = False) -> Mapping[str, JsonValue] | None:
|
||||
if openai_sdk and isinstance(value, Mapping):
|
||||
body: Final = {
|
||||
key: item for key, item in value.items() if key not in ("extra_headers", "extra_query", "extra_body")
|
||||
}
|
||||
extra_body: Final = value.get("extra_body")
|
||||
return classifier_input_snapshot({**body, **extra_body} if isinstance(extra_body, Mapping) else body)
|
||||
try:
|
||||
return (
|
||||
_JSON_OBJECT.validate_json(value)
|
||||
if isinstance(value, (str, bytes))
|
||||
else _JSON_OBJECT.validate_python(value)
|
||||
)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def is_classifier_call(call_type: str, params: Mapping[str, object]) -> bool:
|
||||
return call_type in ("completion", "acompletion") and any(
|
||||
isinstance(metadata := params.get(key), Mapping)
|
||||
and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == AUTOROUTER_CLASSIFIER_CALL_ORIGIN
|
||||
for key in ("metadata", "litellm_metadata")
|
||||
)
|
||||
|
||||
|
||||
def masked_originating_request(request_kwargs: Mapping[str, object] | None) -> Mapping[str, JsonValue] | None:
|
||||
request: Final = (request_kwargs or {}).get("proxy_server_request")
|
||||
body: Final = request.get("body") if isinstance(request, Mapping) else None
|
||||
if not isinstance(body, Mapping):
|
||||
return None
|
||||
serializable: Final = classifier_input_snapshot(safe_dumps(body))
|
||||
return classifier_input_snapshot(redact_credentials_in_payload(serializable)) if serializable is not None else None
|
||||
|
||||
|
||||
def classifier_audit_fields(payload: Mapping[str, object]) -> ClassifierAudit:
|
||||
classifier_input: Final = classifier_input_snapshot(payload.get("classifier_input"))
|
||||
originating_request: Final = classifier_input_snapshot(payload.get("originating_request_masked"))
|
||||
return {
|
||||
**(ClassifierAudit(classifier_input=classifier_input) if classifier_input is not None else {}),
|
||||
**(ClassifierAudit(originating_request_masked=originating_request) if originating_request is not None else {}),
|
||||
}
|
||||
|
||||
|
||||
def without_classifier_audit(payload: Mapping[str, object]) -> dict[str, object]:
|
||||
return {key: value for key, value in payload.items() if key not in CLASSIFIER_AUDIT_FIELDS}
|
||||
|
|
@ -17,7 +17,7 @@ from types import MappingProxyType, TracebackType
|
|||
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast
|
||||
|
||||
from httpx import Response
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, JsonValue
|
||||
|
||||
import litellm
|
||||
from litellm import (
|
||||
|
|
@ -64,6 +64,11 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.integrations.deepeval.deepeval import DeepEvalLogger
|
||||
from litellm.integrations.mlflow import MlflowLogger
|
||||
from litellm.integrations.sqs import SQSLogger
|
||||
from litellm.litellm_core_utils.classifier_logging import (
|
||||
classifier_audit_fields,
|
||||
classifier_input_snapshot,
|
||||
is_classifier_call,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
|
||||
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
|
|
@ -89,6 +94,7 @@ from litellm.litellm_core_utils.redact_messages import (
|
|||
redact_message_input_output_from_custom_logger,
|
||||
redact_message_input_output_from_logging,
|
||||
redact_streaming_responses_for_custom_logger,
|
||||
should_redact_message_logging,
|
||||
)
|
||||
from litellm.llms.base_llm.ocr.transformation import OCRResponse
|
||||
from litellm.llms.base_llm.search.transformation import SearchResponse
|
||||
|
|
@ -473,6 +479,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
stream_options = None
|
||||
litellm_request_debug: bool = False
|
||||
streamed_anthropic_message_id: str | None = None
|
||||
classifier_input: Mapping[str, JsonValue] | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -1211,6 +1218,14 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model_call_details["api_key"] = api_key
|
||||
self.model_call_details["additional_args"] = additional_args
|
||||
self.model_call_details["log_event_type"] = "pre_api_call"
|
||||
if is_classifier_call(self.call_type, self.model_call_details.get("litellm_params") or {}):
|
||||
self.classifier_input = (
|
||||
None
|
||||
if should_redact_message_logging(self.model_call_details)
|
||||
else classifier_input_snapshot(
|
||||
additional_args.get("complete_input_dict"), openai_sdk=additional_args.get("openai_sdk") is True
|
||||
)
|
||||
)
|
||||
if model: # if model name was changes pre-call, overwrite the initial model call name with the new one
|
||||
self.model_call_details["model"] = model
|
||||
self.model_call_details["litellm_params"]["api_base"] = self._get_masked_api_base(
|
||||
|
|
@ -6293,6 +6308,16 @@ def get_standard_logging_object_payload(
|
|||
)
|
||||
|
||||
payload: Final[StandardLoggingPayload] = StandardLoggingPayload(
|
||||
**(
|
||||
classifier_audit_fields(
|
||||
{
|
||||
"classifier_input": logging_obj.classifier_input,
|
||||
"originating_request_masked": proxy_server_request.get("originating_request_masked"),
|
||||
}
|
||||
)
|
||||
if is_classifier_call(call_type or "", litellm_params) and not should_redact_message_logging(kwargs)
|
||||
else {}
|
||||
),
|
||||
id=str(id),
|
||||
litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),
|
||||
trace_id=StandardLoggingPayloadSetup.get_standard_logging_payload_trace_id(
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
import litellm
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
)
|
||||
|
|
@ -183,6 +184,9 @@ def _redact_standard_logging_object(model_call_details: dict):
|
|||
|
||||
redacted_str: Final = REDACTED_BY_LITELLM
|
||||
|
||||
for field in CLASSIFIER_AUDIT_FIELDS:
|
||||
standard_logging_object.pop(field, None)
|
||||
|
||||
if standard_logging_object.get("messages") is not None:
|
||||
standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}]
|
||||
|
||||
|
|
@ -254,6 +258,12 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
|
|||
copy via redact_streaming_responses_for_custom_logger instead.
|
||||
"""
|
||||
# Redact model_call_details
|
||||
for field in CLASSIFIER_AUDIT_FIELDS:
|
||||
model_call_details.pop(field, None)
|
||||
params: Final = model_call_details.get("litellm_params")
|
||||
request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None
|
||||
if isinstance(params, dict) and isinstance(request, Mapping):
|
||||
model_call_details["litellm_params"] = {**params, "proxy_server_request": without_classifier_audit(request)}
|
||||
model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
|
||||
model_call_details["prompt"] = ""
|
||||
model_call_details["input"] = ""
|
||||
|
|
|
|||
|
|
@ -797,6 +797,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
"api_base": openai_client._base_url._uri_reference,
|
||||
"acompletion": acompletion,
|
||||
"complete_input_dict": data,
|
||||
"openai_sdk": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -938,6 +939,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
"api_base": openai_aclient._base_url._uri_reference,
|
||||
"acompletion": True,
|
||||
"complete_input_dict": data,
|
||||
"openai_sdk": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ from typing_extensions import ReadOnly
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME
|
||||
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME
|
||||
from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields, classifier_input_snapshot
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy._types import ProviderBudgetResponse, ProviderBudgetResponseObject
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
|
@ -3099,7 +3100,11 @@ async def _resolve_request_response_payload(
|
|||
proxy_server_request: Final = row.get("proxy_server_request")
|
||||
|
||||
pg_payload: Final = RequestResponsePayload(messages, response, proxy_server_request)
|
||||
if (
|
||||
stored_request: Final = classifier_input_snapshot(proxy_server_request)
|
||||
truncated_audit: Final = bool(stored_request and classifier_audit_fields(stored_request)) and (
|
||||
LITELLM_TRUNCATED_PAYLOAD_FIELD in str(proxy_server_request)
|
||||
)
|
||||
if not truncated_audit and (
|
||||
_spend_log_field_has_content(messages)
|
||||
or _spend_log_field_has_content(response)
|
||||
or _spend_log_field_has_content(proxy_server_request)
|
||||
|
|
@ -3124,10 +3129,19 @@ async def _resolve_request_response_payload(
|
|||
if payload is None:
|
||||
return pg_payload
|
||||
|
||||
cold_audit: Final = classifier_audit_fields(payload)
|
||||
resolved_request: Final = (
|
||||
{**(classifier_input_snapshot(payload.get("proxy_server_request")) or stored_request or {}), **cold_audit}
|
||||
if cold_audit
|
||||
else payload.get("proxy_server_request")
|
||||
)
|
||||
if truncated_audit:
|
||||
return RequestResponsePayload(messages, response, resolved_request if cold_audit else proxy_server_request)
|
||||
|
||||
return RequestResponsePayload(
|
||||
messages=payload.get("messages"),
|
||||
response=payload.get("response"),
|
||||
proxy_server_request=payload.get("proxy_server_request"),
|
||||
proxy_server_request=resolved_request,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from litellm.constants import (
|
|||
from litellm.constants import (
|
||||
MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB,
|
||||
)
|
||||
from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
get_litellm_metadata_from_kwargs,
|
||||
reconstruct_model_name,
|
||||
|
|
@ -1251,6 +1252,10 @@ def _get_proxy_server_request_for_spend_logs_payload(
|
|||
if _proxy_server_request is not None:
|
||||
_request_body = _proxy_server_request.get("body", {}) or {}
|
||||
|
||||
standard_payload: Final = (kwargs or {}).get("standard_logging_object")
|
||||
if isinstance(standard_payload, Mapping):
|
||||
_request_body = {**_request_body, **classifier_audit_fields(standard_payload)}
|
||||
|
||||
if kwargs is not None:
|
||||
realtime_tools: Final = kwargs.get("realtime_tools")
|
||||
if realtime_tools:
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from litellm.constants import (
|
|||
SESSION_ID_GENERATED_METADATA_KEY,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.classifier_logging import masked_originating_request
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
_get_parent_otel_span_from_kwargs,
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
|
|
@ -1924,11 +1925,8 @@ class ComplexityRouter(CustomLogger):
|
|||
Call the configured classifier model with a system/user role split and prior-turn context.
|
||||
|
||||
Builds a structured classification prompt with:
|
||||
- System message: the stable classifier rubric AND the caller's own system prompt (task
|
||||
constraints). This is the largest, most repeated part of the call, so keeping it in the
|
||||
system role lets the provider prompt-cache it across a session's classifier calls.
|
||||
- User message: the variable payload -- a few prior user turns for context and the current
|
||||
ask to classify.
|
||||
- System message: the stable classifier rubric.
|
||||
- User message: the caller's system prompt quoted as task context, prior turns, and the current ask.
|
||||
|
||||
Args:
|
||||
prompt: The current user ask text (already extracted as the real human ask, not tool results)
|
||||
|
|
@ -2005,12 +2003,13 @@ class ComplexityRouter(CustomLogger):
|
|||
classifier_call_params = MappingProxyType({"reasoning_effort": llm_config.reasoning_effort})
|
||||
|
||||
proxy_server_request: Final = {
|
||||
"originating_request_masked": masked_originating_request(request_kwargs),
|
||||
"body": {
|
||||
"model": llm_config.model,
|
||||
"messages": messages_for_call,
|
||||
"response_format": response_format,
|
||||
**classifier_call_params,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from pydantic import (
|
|||
BaseModel,
|
||||
ConfigDict,
|
||||
Field,
|
||||
JsonValue,
|
||||
PrivateAttr,
|
||||
SkipValidation,
|
||||
field_serializer,
|
||||
|
|
@ -3373,7 +3374,12 @@ class StandardAuditLogPayload(TypedDict):
|
|||
updated_values: str | None
|
||||
|
||||
|
||||
class StandardLoggingPayload(TypedDict):
|
||||
class ClassifierAudit(TypedDict, total=False):
|
||||
classifier_input: ReadOnly[Mapping[str, JsonValue]]
|
||||
originating_request_masked: ReadOnly[Mapping[str, JsonValue]]
|
||||
|
||||
|
||||
class StandardLoggingPayload(ClassifierAudit):
|
||||
id: str
|
||||
trace_id: str # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries)
|
||||
session_id: str # End-user/conversation session id (litellm_session_id), independent of trace_id
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
# Reliability & Performance (behavior features). Grounded in litellm/router.py + router_strategy/ + router_utils/.
|
||||
- {id: reliability.routing.classifier_audit.separates_provider_input_and_source, module: reliability, tier: P1, behavior: routing, variant: classifier_audit, assertions: [separates_provider_input_and_source], exercised_on: [chat_completions, messages, responses], source: "litellm/litellm_core_utils/classifier_logging.py", rationale: "Classifier spend details distinguish provider input from the credential-masked originating request across all three request surfaces"}
|
||||
- {id: reliability.fallback.5xx.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: "5xx", assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:2024", rationale: "Reroute on provider 5xx to alternate deployment"}
|
||||
- {id: reliability.fallback.context_window.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: context_window, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6108", rationale: "Fallback when model exceeds context limit"}
|
||||
- {id: reliability.fallback.content_policy.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: content_policy, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6023", rationale: "Reroute on content-policy violation"}
|
||||
|
|
|
|||
143
tests/e2e/router/test_classifier_audit_e2e.py
Normal file
143
tests/e2e/router/test_classifier_audit_e2e.py
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
import json
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, Field, JsonValue, TypeAdapter
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import AnthropicHeaders, NoBody, unwrap
|
||||
from models import ChatMessage, KeyGenerateBody, LiteLLMParamsBody
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
||||
class AuditMetadata(BaseModel):
|
||||
source_marker: str
|
||||
authorization: str = "synthetic-audit-secret"
|
||||
|
||||
|
||||
class AuditHeaders(AnthropicHeaders):
|
||||
enable_redaction: str | None = Field(default=None, serialization_alias="x-litellm-enable-message-redaction")
|
||||
|
||||
|
||||
class AuditRequest(BaseModel):
|
||||
model: str
|
||||
messages: list[ChatMessage] | None = None
|
||||
system: str | None = None
|
||||
instructions: str | None = None
|
||||
input: str | None = None
|
||||
max_tokens: int | None = None
|
||||
max_output_tokens: int | None = None
|
||||
metadata: AuditMetadata | None = None
|
||||
litellm_metadata: AuditMetadata | None = None
|
||||
|
||||
|
||||
class AuditResponse(BaseModel):
|
||||
id: str
|
||||
|
||||
|
||||
class AuditDetail(BaseModel):
|
||||
proxy_server_request: dict[str, JsonValue] | str | None = None
|
||||
response: dict[str, JsonValue] | str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AuditDeployment:
|
||||
alias: str
|
||||
key: str
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def audit_deployment(proxy: ProxyClient, provider: str) -> Iterator[AuditDeployment]:
|
||||
marker: Final = unique_marker()
|
||||
classifier: Final = f"audit-classifier-{marker}"
|
||||
target: Final = f"audit-target-{marker}"
|
||||
alias: Final = f"audit-router-{marker}"
|
||||
model: Final = os.environ.get(
|
||||
f"E2E_CHEAP_{provider.upper()}_MODEL", "gpt-5.6" if provider == "openai" else "claude-haiku-4-5"
|
||||
)
|
||||
params: Final = LiteLLMParamsBody(
|
||||
model=f"{provider}/{model}",
|
||||
api_key=os.environ.get(f"{provider.upper()}_API_KEY") or f"os.environ/{provider.upper()}_API_KEY",
|
||||
api_base=os.environ.get(f"{provider.upper()}_API_BASE"),
|
||||
)
|
||||
with ExitStack() as stack:
|
||||
for name in (classifier, target):
|
||||
stack.callback(proxy.delete_model, proxy.create_model(name, params))
|
||||
stack.callback(proxy.delete_model, proxy.create_model(alias, LiteLLMParamsBody(
|
||||
model="auto_router/complexity_router",
|
||||
complexity_router_config={
|
||||
"classifier_type": "llm",
|
||||
"classifier_llm_config": {"model": classifier, "timeout_ms": 30000},
|
||||
"tiers": {tier: target for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")},
|
||||
},
|
||||
)))
|
||||
key: Final = proxy.generate_key(KeyGenerateBody(models=[alias, classifier, target]))
|
||||
stack.callback(proxy.delete_key, key)
|
||||
yield AuditDeployment(alias, key)
|
||||
|
||||
|
||||
class TestClassifierAudit:
|
||||
@pytest.mark.covers(
|
||||
"reliability.routing.classifier_audit.separates_provider_input_and_source",
|
||||
exercised_on=("chat_completions", "messages", "responses"),
|
||||
)
|
||||
@pytest.mark.parametrize("surface", ["chat_completions", "messages", "responses"])
|
||||
@pytest.mark.parametrize("provider", ["openai", "anthropic"])
|
||||
@pytest.mark.parametrize("redact", [False, True])
|
||||
def test_classifier_audit_separates_input_and_masked_source(
|
||||
self, proxy: ProxyClient, audit_deployment: AuditDeployment, surface: Literal["chat_completions", "messages", "responses"],
|
||||
redact: bool,
|
||||
) -> None:
|
||||
marker: Final = unique_marker()
|
||||
source_marker: Final = f"source-only-{marker}"
|
||||
prompt: Final = f"Reply with hello. Request label {marker}"
|
||||
metadata: Final = AuditMetadata(source_marker=source_marker)
|
||||
body: Final = AuditRequest(
|
||||
model=audit_deployment.alias,
|
||||
messages=[ChatMessage(role="user", content=prompt)] if surface != "responses" else None,
|
||||
input=prompt if surface == "responses" else None,
|
||||
system="Be concise" if surface == "messages" else None,
|
||||
instructions="Be concise" if surface == "responses" else None,
|
||||
max_tokens=128 if surface != "responses" else None,
|
||||
max_output_tokens=128 if surface == "responses" else None,
|
||||
metadata=metadata if surface != "responses" else None,
|
||||
litellm_metadata=metadata if surface == "responses" else None,
|
||||
)
|
||||
path: Final = {"chat_completions": "/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}[surface]
|
||||
response: Final = unwrap(proxy.transport.post(
|
||||
path, headers=AuditHeaders(authorization=f"Bearer {audit_deployment.key}", enable_redaction="true" if redact else None),
|
||||
json=body, response_type=AuditResponse,
|
||||
))
|
||||
assert response.id
|
||||
rows: Final = proxy.poll_logs_for_key(audit_deployment.key, min_rows=2)
|
||||
assert len(rows) == 2, "Expected a classifier spend row and a routed response spend row"
|
||||
details: Final = tuple(
|
||||
unwrap(proxy.transport.get(
|
||||
f"/spend/logs/ui/{row.request_id}", headers=proxy.transport.master,
|
||||
params=NoBody(), response_type=AuditDetail,
|
||||
)) for row in rows
|
||||
)
|
||||
adapter: Final = TypeAdapter(dict[str, JsonValue])
|
||||
requests: Final = tuple(
|
||||
adapter.validate_json(detail.proxy_server_request) if isinstance(detail.proxy_server_request, str)
|
||||
else detail.proxy_server_request or {} for detail in details
|
||||
)
|
||||
audits: Final = tuple(item for item in requests if "classifier_input" in item)
|
||||
if redact:
|
||||
assert audits == ()
|
||||
assert all("originating_request_masked" not in item for item in requests)
|
||||
return
|
||||
assert len(audits) == 1, "The audit belongs only to the classifier call"
|
||||
audit: Final = audits[0]
|
||||
assert marker in json.dumps(audit["classifier_input"])
|
||||
assert source_marker not in json.dumps(audit["classifier_input"])
|
||||
assert source_marker in json.dumps(audit["originating_request_masked"])
|
||||
assert "synthetic-audit-secret" not in json.dumps(audit)
|
||||
assert audit_deployment.alias in json.dumps(audit["originating_request_masked"])
|
||||
assert any("tier" in str(detail.response) for detail in details)
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot, masked_originating_request
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoded", [False, True])
|
||||
def test_classifier_snapshot_preserves_provider_shape_and_is_independent(encoded: bool) -> None:
|
||||
import json
|
||||
|
||||
provider_body: Final = {"system": [{"text": "rubric"}], "messages": [{"role": "user", "content": "ask"}]}
|
||||
snapshot: Final = classifier_input_snapshot(json.dumps(provider_body) if encoded else provider_body)
|
||||
assert snapshot == provider_body
|
||||
provider_body["messages"][0]["content"] = "later mutation"
|
||||
assert snapshot == {"system": [{"text": "rubric"}], "messages": [{"role": "user", "content": "ask"}]}
|
||||
|
||||
|
||||
def test_originating_snapshot_masks_nested_credentials_without_altering_source() -> None:
|
||||
body: Final = {
|
||||
"model": "router",
|
||||
"input": [{"type": "message", "role": "user", "content": "source-only"}],
|
||||
"api_key": "short",
|
||||
"metadata": {"nested": [{"Authorization": "Bearer secret", "access_token": 123}]},
|
||||
}
|
||||
snapshot: Final = masked_originating_request({"proxy_server_request": {"body": body}})
|
||||
assert snapshot is not None
|
||||
assert snapshot["model"] == "router"
|
||||
assert snapshot["input"] == body["input"]
|
||||
assert snapshot["api_key"] == "REDACTED"
|
||||
assert snapshot["metadata"] == {"nested": [{"Authorization": "REDACTED", "access_token": "REDACTED"}]}
|
||||
assert body["api_key"] == "short"
|
||||
assert body["metadata"]["nested"][0]["Authorization"] == "Bearer secret"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, "not-json", [], {"messages": object()}])
|
||||
def test_invalid_provider_payload_is_not_reported_as_captured(value: object) -> None:
|
||||
assert classifier_input_snapshot(value) is None
|
||||
|
|
@ -3,7 +3,7 @@ import contextlib
|
|||
import datetime
|
||||
import os
|
||||
import sys
|
||||
from typing import Literal
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -6801,3 +6801,122 @@ def test_get_error_information_redacts_provider_key_from_upstream_url():
|
|||
assert "REDACTED" in result["traceback"]
|
||||
assert "REDACTED" in result["error_message"]
|
||||
assert result["error_code"] == "400"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("provider", ["openai", "anthropic", "bedrock"])
|
||||
async def test_classifier_audit_matches_provider_transport(provider: str) -> None:
|
||||
import json
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
||||
outbound: Final = asyncio.Queue()
|
||||
logs: Final = asyncio.Queue()
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
outbound.put_nowait(json.loads(request.content))
|
||||
content: Final = '{"tier":"SIMPLE"}'
|
||||
if provider == "anthropic":
|
||||
return httpx.Response(200, json={
|
||||
"id": "msg-audit", "type": "message", "role": "assistant", "model": "claude-haiku-4-5",
|
||||
"content": [{"type": "text", "text": content}], "stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
})
|
||||
if provider == "bedrock":
|
||||
return httpx.Response(200, json={
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": content}]}},
|
||||
"stopReason": "end_turn", "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15},
|
||||
"metrics": {"latencyMs": 1},
|
||||
})
|
||||
return httpx.Response(200, json={
|
||||
"id": "chatcmpl-audit", "object": "chat.completion", "created": 0, "model": "gpt-5.6",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
})
|
||||
|
||||
async def capture(kwargs, response_obj, start_time, end_time):
|
||||
logs.put_nowait(kwargs["standard_logging_object"])
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client:
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.close()
|
||||
handler.client = http_client
|
||||
client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client) if provider == "openai" else handler
|
||||
model: Final = {
|
||||
"openai": "openai/gpt-5.6",
|
||||
"anthropic": "anthropic/claude-haiku-4-5",
|
||||
"bedrock": "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
}[provider]
|
||||
|
||||
async def run(marker: str) -> None:
|
||||
await litellm.acompletion(
|
||||
model=model, api_key="transport-only", client=client, max_tokens=128,
|
||||
aws_access_key_id="transport-only", aws_secret_access_key="transport-only", aws_region_name="us-east-1",
|
||||
messages=[{"role": "system", "content": "classifier-rubric"}, {"role": "user", "content": marker}],
|
||||
metadata={"internal_call_origin": "autorouter_classifier"},
|
||||
proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}},
|
||||
success_callback=[capture], num_retries=0,
|
||||
**({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}}
|
||||
if provider == "openai" else {}),
|
||||
)
|
||||
|
||||
await asyncio.gather(run("request-one"), run("request-two"))
|
||||
requests: Final = await asyncio.wait_for(asyncio.gather(outbound.get(), outbound.get()), timeout=10)
|
||||
payloads: Final = await asyncio.wait_for(asyncio.gather(logs.get(), logs.get()), timeout=10)
|
||||
for payload in payloads:
|
||||
snapshot: Final = payload["classifier_input"]
|
||||
assert snapshot in requests
|
||||
assert "source-only" not in json.dumps(snapshot)
|
||||
assert "classifier-rubric" in json.dumps(snapshot)
|
||||
assert "transport-only" not in json.dumps(snapshot)
|
||||
assert "header-only-secret" not in json.dumps(snapshot)
|
||||
assert "SIMPLE" in json.dumps(payload["response"])
|
||||
marker: Final = "request-one" if "request-one" in json.dumps(snapshot) else "request-two"
|
||||
assert payload["originating_request_masked"] == {"input": f"source-only-{marker}"}
|
||||
assert classifier_input_snapshot(snapshot) is not None
|
||||
if provider != "openai":
|
||||
assert all("system" in request for request in requests)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("redaction", ["none", "global", "request", "header"])
|
||||
@pytest.mark.parametrize("status", ["success", "failure"])
|
||||
def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_obj, monkeypatch, redaction, status):
|
||||
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
|
||||
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", redaction == "global")
|
||||
params: Final = {
|
||||
"metadata": {"internal_call_origin": "autorouter_classifier", **(
|
||||
{"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {}
|
||||
)},
|
||||
"proxy_server_request": {"body": {}, "originating_request_masked": {"input": "source-only"}},
|
||||
}
|
||||
logging_obj.model_call_details["litellm_params"] = params
|
||||
logging_obj.model_call_details["standard_callback_dynamic_params"] = (
|
||||
{"turn_off_message_logging": True} if redaction == "request" else {}
|
||||
)
|
||||
logging_obj.pre_call(
|
||||
input=[], api_key=None, additional_args={"complete_input_dict": {"system": "rubric", "messages": []}}
|
||||
)
|
||||
now: Final = datetime.datetime.now()
|
||||
payload: Final = get_standard_logging_object_payload(
|
||||
kwargs={**logging_obj.model_call_details, "call_type": "completion"}, init_response_obj={},
|
||||
start_time=now, end_time=now, logging_obj=logging_obj, status=status,
|
||||
)
|
||||
assert payload is not None
|
||||
if redaction == "none":
|
||||
assert payload["classifier_input"] == {"system": "rubric", "messages": []}
|
||||
assert payload["originating_request_masked"] == {"input": "source-only"}
|
||||
else:
|
||||
assert "classifier_input" not in payload
|
||||
assert "originating_request_masked" not in payload
|
||||
|
||||
|
||||
@pytest.mark.parametrize("call_type,origin", [("completion", None), ("aembedding", "autorouter_classifier")])
|
||||
def test_classifier_audit_is_not_added_to_other_calls(logging_obj, call_type, origin):
|
||||
logging_obj.call_type = call_type
|
||||
logging_obj.model_call_details["litellm_params"] = {"metadata": {"internal_call_origin": origin}}
|
||||
logging_obj.pre_call(input=[], api_key=None, additional_args={"complete_input_dict": {"input": "embedding"}})
|
||||
assert logging_obj.classifier_input is None
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ but litellm_params["litellm_metadata"] is None.
|
|||
"""
|
||||
|
||||
import threading
|
||||
from typing import Final
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
|
@ -776,6 +777,29 @@ class TestPerformRedaction:
|
|||
|
||||
assert response_obj.choices[0].message.content == "secret content"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("callback_only", [False, True])
|
||||
def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callback_only: bool) -> None:
|
||||
audit: Final = {"classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}}
|
||||
details: Final = {
|
||||
"standard_logging_object": {**audit, "messages": [], "response": {}},
|
||||
"litellm_params": {"proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]}},
|
||||
}
|
||||
logger: Final = CustomLogger()
|
||||
logger.turn_off_message_logging = True
|
||||
if callback_only:
|
||||
redacted: Final = logger.redact_standard_logging_payload_from_model_call_details(details)
|
||||
assert "classifier_input" not in redacted["standard_logging_object"]
|
||||
assert "originating_request_masked" not in redacted["standard_logging_object"]
|
||||
assert "originating_request_masked" not in redacted["litellm_params"]["proxy_server_request"]
|
||||
assert details["standard_logging_object"]["classifier_input"] == audit["classifier_input"]
|
||||
assert details["litellm_params"]["proxy_server_request"]["originating_request_masked"] == audit["originating_request_masked"]
|
||||
else:
|
||||
perform_redaction(details, result=None)
|
||||
assert "classifier_input" not in details["standard_logging_object"]
|
||||
assert "originating_request_masked" not in details["standard_logging_object"]
|
||||
assert "originating_request_masked" not in details["litellm_params"]["proxy_server_request"]
|
||||
|
||||
def test_unredactable_result_is_not_deepcopied(self):
|
||||
"""A result shape no branch can redact must not be deepcopied.
|
||||
|
||||
|
|
|
|||
|
|
@ -5703,6 +5703,29 @@ def _cold_storage_handler(payload):
|
|||
return ColdStorageHandler(cold_storage_logger=logger), logger
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("cold_has_audit", [False, True])
|
||||
async def test_resolve_payload_recovers_truncated_classifier_audit_without_losing_existing_fields(cold_has_audit):
|
||||
full_audit = {"classifier_input": {"system": "full rubric"}, "originating_request_masked": {"input": "source"}}
|
||||
truncated_request = {"model": "classifier", "classifier_input": {"system": "litellm_truncated"}}
|
||||
handler, logger = _cold_storage_handler({
|
||||
"proxy_server_request": {"body": {}}, **(full_audit if cold_has_audit else {}),
|
||||
})
|
||||
row = {
|
||||
"messages": '[{"role":"user","content":"ask"}]', "response": '{"tier":"SIMPLE"}',
|
||||
"proxy_server_request": json.dumps(truncated_request), "metadata": {"cold_storage_object_key": "k/audit.json"},
|
||||
}
|
||||
resolved = await spend_management_endpoints._resolve_request_response_payload(row, cold_storage_handler=handler)
|
||||
assert logger.requested_object_keys == ["k/audit.json"]
|
||||
assert resolved.messages == row["messages"]
|
||||
assert resolved.response == row["response"]
|
||||
if cold_has_audit:
|
||||
assert resolved.proxy_server_request["classifier_input"] == full_audit["classifier_input"]
|
||||
assert resolved.proxy_server_request["originating_request_masked"] == full_audit["originating_request_masked"]
|
||||
else:
|
||||
assert resolved.proxy_server_request == row["proxy_server_request"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value, expected",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -67,6 +67,30 @@ def _get_additional_usage_values_for_usage(usage: litellm.Usage) -> dict:
|
|||
return metadata["additional_usage_values"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)])
|
||||
def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch, store_prompts, redact):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": store_prompts})
|
||||
audit: Final = {
|
||||
"classifier_input": {"system": "rubric" * 1000, "messages": [{"role": "user", "content": "ask"}]},
|
||||
"originating_request_masked": {"input": "source-only", "api_key": "REDACTED"},
|
||||
}
|
||||
stored: Final = json.loads(_get_proxy_server_request_for_spend_logs_payload(
|
||||
metadata={}, litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}},
|
||||
kwargs={"standard_logging_object": audit, "standard_callback_dynamic_params": {"turn_off_message_logging": redact}},
|
||||
))
|
||||
if not store_prompts or redact:
|
||||
assert "classifier_input" not in stored
|
||||
assert "originating_request_masked" not in stored
|
||||
else:
|
||||
assert stored["classifier_input"]["messages"] == audit["classifier_input"]["messages"]
|
||||
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in json.dumps(stored["classifier_input"])
|
||||
assert stored["originating_request_masked"]["input"] == "source-only"
|
||||
assert stored["model"] == "classifier"
|
||||
assert audit["classifier_input"]["system"] == "rubric" * 1000
|
||||
|
||||
|
||||
def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_tokens():
|
||||
additional_usage_values = _get_additional_usage_values_for_usage(
|
||||
litellm.Usage(
|
||||
|
|
|
|||
|
|
@ -2940,6 +2940,29 @@ class TestLLMClassifier:
|
|||
"REASONING",
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("source_body", [
|
||||
{"model": "router", "messages": [{"role": "user", "content": "source-only"}]},
|
||||
{"model": "router", "system": "source-only", "messages": [{"role": "user", "content": "ask"}]},
|
||||
{"model": "router", "instructions": "source-only", "input": "ask"},
|
||||
])
|
||||
async def test_classifier_source_is_masked_and_separate_from_provider_input(
|
||||
self, llm_complexity_router, mock_router_instance, source_body
|
||||
):
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
|
||||
outcome = await llm_complexity_router.aclassify(
|
||||
"classify-this-ask", request_kwargs={"proxy_server_request": {
|
||||
"body": {**source_body, "metadata": {"authorization": "source-secret"}}
|
||||
}}
|
||||
)
|
||||
assert outcome.cause == "llm_classifier"
|
||||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
source = call_kwargs["proxy_server_request"]["originating_request_masked"]
|
||||
assert source == {**source_body, "metadata": {"authorization": "REDACTED"}}
|
||||
assert "source-only" not in str(call_kwargs["messages"])
|
||||
assert "source-only" not in str(call_kwargs["proxy_server_request"]["body"])
|
||||
assert "classify-this-ask" in str(call_kwargs["messages"])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("reasoning_effort", [None, "none", "low"], ids=["omitted", "none", "low"])
|
||||
async def test_classifier_reasoning_effort_reaches_only_classifier_call(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
import { render, screen, within } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { ClassifierAuditView } from "./ClassifierAuditView";
|
||||
|
||||
vi.mock("./JsonViewer", () => ({
|
||||
JsonViewer: ({ data }: { data: unknown }) => <pre>{JSON.stringify(data)}</pre>,
|
||||
}));
|
||||
|
||||
describe("ClassifierAuditView", () => {
|
||||
it("separates and copies the provider input, source request, and returned verdict", async () => {
|
||||
const user = userEvent.setup();
|
||||
const input = { system: "classification rubric", messages: [{ role: "user", content: "classify this" }] };
|
||||
render(
|
||||
<ClassifierAuditView
|
||||
request={{ classifier_input: input, originating_request_masked: { input: "source-only", api_key: "REDACTED" } }}
|
||||
response={{ tier: "SIMPLE", reason: "a greeting" }}
|
||||
/>,
|
||||
);
|
||||
const classifier = within(screen.getByRole("region", { name: "Classifier input" }));
|
||||
expect(classifier.getByText(/classification rubric/)).toBeInTheDocument();
|
||||
expect(classifier.queryByText(/source-only/)).not.toBeInTheDocument();
|
||||
expect(
|
||||
within(screen.getByRole("region", { name: "Originating request, credentials masked" })).getByText(/source-only/),
|
||||
).toBeInTheDocument();
|
||||
expect(
|
||||
within(screen.getByRole("region", { name: "Classifier response" })).getByText(/a greeting/),
|
||||
).toBeInTheDocument();
|
||||
await user.click(classifier.getByRole("button", { name: "Copy Classifier input" }));
|
||||
expect(await navigator.clipboard.readText()).toBe(JSON.stringify(input, null, 2));
|
||||
});
|
||||
|
||||
it("does not present legacy source messages as captured classifier input", () => {
|
||||
render(<ClassifierAuditView request={{ messages: [{ content: "legacy source" }] }} response={undefined} />);
|
||||
expect(screen.getAllByText("Not captured or message logging disabled")).toHaveLength(3);
|
||||
expect(screen.queryByRole("button", { name: "Copy Classifier input" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("labels truncated input without marking a complete source request as truncated", () => {
|
||||
render(
|
||||
<ClassifierAuditView
|
||||
request={{
|
||||
classifier_input: { system: "partial rubric...litellm_truncated" },
|
||||
originating_request_masked: { input: "source" },
|
||||
}}
|
||||
response={{ tier: "SIMPLE" }}
|
||||
/>,
|
||||
);
|
||||
expect(within(screen.getByRole("region", { name: "Classifier input" })).getByRole("status")).toHaveTextContent(
|
||||
"This stored copy is truncated",
|
||||
);
|
||||
expect(
|
||||
within(screen.getByRole("region", { name: "Originating request, credentials masked" })).queryByRole("status"),
|
||||
).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
import CopyButton from "@/components/shared/CopyButton";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import type { ReactNode } from "react";
|
||||
import { JsonViewer } from "./JsonViewer";
|
||||
|
||||
interface ClassifierAuditViewProps {
|
||||
request: Record<string, unknown>;
|
||||
response: unknown;
|
||||
}
|
||||
|
||||
export function ClassifierAuditView({ request, response }: ClassifierAuditViewProps) {
|
||||
return (
|
||||
<div className="mb-6 space-y-4">
|
||||
<AuditField title="Classifier input" value={request.classifier_input}>
|
||||
Provider request payload. A cached call or disabled message logging may have no capture.
|
||||
</AuditField>
|
||||
<AuditField title="Originating request, credentials masked" value={request.originating_request_masked}>
|
||||
Comparison only. This source request was not appended to the classifier input.
|
||||
</AuditField>
|
||||
<AuditField title="Classifier response" value={response}>
|
||||
The returned verdict and any explanation supplied by the classifier. Later routing rules may change the tier.
|
||||
</AuditField>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function AuditField({ title, value, children }: { title: string; value: unknown; children: ReactNode }) {
|
||||
const serialized = JSON.stringify(value);
|
||||
const truncated = serialized?.includes("litellm_truncated") ?? false;
|
||||
|
||||
return (
|
||||
<Card size="sm" role="region" aria-label={title}>
|
||||
<CardHeader>
|
||||
<CardTitle>{title}</CardTitle>
|
||||
{value != null && <CopyButton value={JSON.stringify(value, null, 2)} label={`Copy ${title}`} />}
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<p className="mb-3 text-sm text-muted-foreground">{children}</p>
|
||||
{truncated && (
|
||||
<p role="status" className="mb-3 text-sm text-warning">
|
||||
This stored copy is truncated. The complete payload is unavailable from the configured log storage.
|
||||
</p>
|
||||
)}
|
||||
{value == null ? (
|
||||
<p className="text-sm text-muted-foreground">Not captured or message logging disabled</p>
|
||||
) : (
|
||||
<JsonViewer data={value} mode="formatted" />
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
|
@ -48,6 +48,8 @@ import {
|
|||
} from "./constants";
|
||||
import { ToolsSection } from "../ToolsSection";
|
||||
import { PrettyMessagesView } from "./PrettyMessagesView";
|
||||
import { ClassifierAuditView } from "./ClassifierAuditView";
|
||||
import { AUTOROUTER_CLASSIFIER_ORIGIN } from "./ClassifyTag";
|
||||
|
||||
export interface LogDetailContentProps {
|
||||
logEntry: LogEntry;
|
||||
|
|
@ -68,6 +70,9 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok
|
|||
const metadata = logEntry.metadata || {};
|
||||
const hasError = metadata.status === "failure";
|
||||
const errorInfo = hasError ? metadata.error_information : null;
|
||||
const isClassifier =
|
||||
metadata.internal_call_origin === AUTOROUTER_CLASSIFIER_ORIGIN &&
|
||||
(logEntry.call_type === "completion" || logEntry.call_type === "acompletion");
|
||||
|
||||
const hasMessages = checkHasMessages(logEntry.messages);
|
||||
const hasResponse = checkHasResponse(logEntry.response);
|
||||
|
|
@ -196,7 +201,11 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok
|
|||
Loading request & response data...
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
) : null}
|
||||
{!isLoadingDetails && isClassifier && (
|
||||
<ClassifierAuditView request={getRawRequest()} response={getFormattedResponse()} />
|
||||
)}
|
||||
{!isLoadingDetails && !isClassifier && (
|
||||
<RequestResponseSection
|
||||
hasResponse={hasResponse}
|
||||
hasError={hasError}
|
||||
|
|
@ -541,22 +550,20 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata:
|
|||
)}
|
||||
|
||||
<DescriptionItem label="Retries">
|
||||
{metadata?.attempted_retries !== undefined && metadata?.attempted_retries !== null ? (
|
||||
metadata.attempted_retries > 0 ? (
|
||||
<>
|
||||
{metadata.attempted_retries}
|
||||
{metadata.max_retries !== undefined && metadata.max_retries !== null
|
||||
? ` / ${metadata.max_retries}`
|
||||
: ""}
|
||||
</>
|
||||
) : (
|
||||
<Badge variant="secondary" className="bg-success/15 text-success">
|
||||
None
|
||||
</Badge>
|
||||
)
|
||||
) : (
|
||||
"-"
|
||||
{metadata?.attempted_retries != null && metadata.attempted_retries > 0 && (
|
||||
<>
|
||||
{metadata.attempted_retries}
|
||||
{metadata.max_retries !== undefined && metadata.max_retries !== null
|
||||
? ` / ${metadata.max_retries}`
|
||||
: ""}
|
||||
</>
|
||||
)}
|
||||
{metadata?.attempted_retries != null && metadata.attempted_retries <= 0 && (
|
||||
<Badge variant="secondary" className="bg-success/15 text-success">
|
||||
None
|
||||
</Badge>
|
||||
)}
|
||||
{metadata?.attempted_retries == null && "-"}
|
||||
</DescriptionItem>
|
||||
|
||||
<DescriptionItem label="Start Time">
|
||||
|
|
@ -602,16 +609,10 @@ function RequestResponseSection({
|
|||
const totalTokens = promptTokens + completionTokens;
|
||||
const costBreakdown = logEntry.metadata?.cost_breakdown;
|
||||
const useCostBreakdown = costBreakdown?.input_cost !== undefined && costBreakdown?.output_cost !== undefined;
|
||||
const inputCost = useCostBreakdown
|
||||
? costBreakdown!.input_cost ?? 0
|
||||
: totalTokens > 0
|
||||
? (totalSpend * promptTokens) / totalTokens
|
||||
: 0;
|
||||
const outputCost = useCostBreakdown
|
||||
? costBreakdown!.output_cost ?? 0
|
||||
: totalTokens > 0
|
||||
? (totalSpend * completionTokens) / totalTokens
|
||||
: 0;
|
||||
const estimatedInputCost = totalTokens > 0 ? (totalSpend * promptTokens) / totalTokens : 0;
|
||||
const estimatedOutputCost = totalTokens > 0 ? (totalSpend * completionTokens) / totalTokens : 0;
|
||||
const inputCost = useCostBreakdown ? costBreakdown!.input_cost ?? 0 : estimatedInputCost;
|
||||
const outputCost = useCostBreakdown ? costBreakdown!.output_cost ?? 0 : estimatedOutputCost;
|
||||
|
||||
return (
|
||||
<div className="bg-card rounded-lg shadow-sm w-full max-w-full overflow-hidden mb-6">
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue