diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 84a9c4b7931..ff4de46aab3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3332,6 +3332,7 @@ class SpendLogsMetadata(TypedDict): mcp_tool_call_metadata: Optional[StandardLoggingMCPToolCall] vector_store_request_metadata: Optional[List[StandardLoggingVectorStoreRequest]] guardrail_information: Optional[List[StandardLoggingGuardrailInformation]] + eval_information: Optional[Any] status: StandardLoggingPayloadStatus proxy_server_request: Optional[str] batch_models: Optional[List[str]] diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index dbad25ffce1..889c781004f 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -107,6 +107,7 @@ def _get_spend_logs_metadata( model_map_information=None, usage_object=None, guardrail_information=None, + eval_information=None, cold_storage_object_key=cold_storage_object_key, litellm_overhead_time_ms=None, attempted_retries=None, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c347956cba7..a212d56c1ae 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2760,6 +2760,29 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False): """Risk score 0-10 indicating how risky the request was (higher = riskier). Computed by the guardrail provider.""" +class EvalVerdict(TypedDict, total=False): + criterion_name: str + score: float # 0-100 + reasoning: str + passed: bool + weight: int # criterion weight (0-100) as configured in the guardrail + + +class StandardLoggingEvalInformation(TypedDict, total=False): + eval_id: Optional[str] + eval_name: str + overall_score: float + passed: bool + judge_model: str + iteration: int + eval_error: Optional[str] + start_time: str + end_time: str + duration: float + verdicts: List[Any] + threshold: Optional[float] + + class GuardrailTracingDetail(TypedDict, total=False): """ Typed fields for guardrail tracing metadata.