From d33e2206ad5650fc04864b4e1fb66cba4472049f Mon Sep 17 00:00:00 2001
From: Ishaan Jaffer <ishaan@berri.ai>
Date: Wed, 22 Apr 2026 16:50:46 -0700
Subject: [PATCH] add EvalCriterion, EvalResult, EvalVerdict,
 StandardLoggingEvalInformation types

---
 litellm/types/utils.py | 52 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index d0bc9b78941..4be98adb310 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -2758,6 +2758,57 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False):
     """Risk score 0-10 indicating how risky the request was (higher = riskier). Computed by the guardrail provider."""
 
 
+class EvalCriterion(TypedDict, total=False):
+    name: str
+    weight: int  # 0-100, weights should sum to 100
+    description: str
+    threshold: Optional[float]  # optional per-criterion minimum score
+
+
+class EvalVerdict(TypedDict, total=False):
+    criterion_name: str
+    score: float  # 0-100
+    reasoning: str
+    passed: bool
+
+
+class StandardLoggingEvalInformation(TypedDict, total=False):
+    eval_id: Optional[str]
+    eval_name: str
+    overall_score: float
+    passed: bool
+    judge_model: str
+    iteration: int
+    eval_error: Optional[str]
+    start_time: str
+    end_time: str
+    duration: float
+
+
+class EvalResult(BaseModel):
+    eval_id: Optional[str] = None
+    eval_name: Optional[str] = None
+    overall_score: float = 0.0
+    verdicts: List[EvalVerdict] = []
+    passed: bool = False
+    judge_model: str = ""
+    iteration: int = 0
+    eval_error: Optional[str] = None
+    raw_judge_response: Optional[dict] = None
+
+    def check_thresholds(self, overall_min: float = 0.0) -> bool:
+        from litellm.exceptions import EvalFailedError
+
+        if self.overall_score < overall_min:
+            raise EvalFailedError(
+                message=f"Eval failed: score {self.overall_score:.1f} < minimum {overall_min}",
+                eval_result=self,
+                model=self.judge_model,
+                llm_provider="",
+            )
+        return True
+
+
 class GuardrailTracingDetail(TypedDict, total=False):
     """
     Typed fields for guardrail tracing metadata.
@@ -2888,6 +2939,7 @@ class StandardLoggingPayload(TypedDict):
     model_parameters: dict
     hidden_params: StandardLoggingHiddenParams
     guardrail_information: Optional[List[StandardLoggingGuardrailInformation]]
+    eval_information: Optional[List[StandardLoggingEvalInformation]]
     standard_built_in_tools_params: Optional[StandardBuiltInToolsParams]