diff --git a/litellm/eval_utils.py b/litellm/eval_utils.py new file mode 100644 index 00000000000..b3c16f251c9 --- /dev/null +++ b/litellm/eval_utils.py @@ -0,0 +1,181 @@ +""" +SDK primitive for LLM-judge evaluation. + +Usage: + result = await litellm.aevaluate( + response=model_response, + criteria=[{"name": "Accuracy", "weight": 100, "description": "Is the answer correct?"}], + judge_model="claude-sonnet-4-6", + ) + result.check_thresholds(overall_min=80) +""" + +import asyncio +import json +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Union + +from litellm.types.utils import EvalCriterion, EvalResult, EvalVerdict + + +_JUDGE_SYSTEM_PROMPT = ( + "You are an expert evaluator. Grade the agent response against each criterion " + "and return ONLY valid JSON — no markdown, no backticks, no extra text.\n\n" + "Response format:\n" + '{"scores": [{"criterion": "criterion name", "score": 0-100, "comment": "brief reason"}], ' + '"overall": 0-100, "verdict": "2-3 sentence summary", ' + '"strongest": "criterion name", "weakest": "criterion name"}' +) + + +def _extract_response_text(response: Any) -> str: + """Extract plain text from a ModelResponse or raw string.""" + if isinstance(response, str): + return response + try: + return response.choices[0].message.content or "" + except (AttributeError, IndexError, TypeError): + try: + return str(response) + except Exception: + return "" + + +def _build_judge_messages( + response_text: str, + criteria: List[EvalCriterion], + system_prompt: Optional[str] = None, +) -> List[Dict]: + """Build the judge messages (system + user) for the LLM-judge call.""" + criteria_lines = "\n".join( + f"{i+1}. {c.get('name', f'Criterion {i+1}')} ({c.get('weight', 0)}%): {c.get('description', '')}" + for i, c in enumerate(criteria) + ) + user_content = ( + f"Grade the following agent response against the rubric.\n\n" + f"AGENT RESPONSE:\n{response_text}\n\n" + f"RUBRIC (score each criterion 0-100):\n{criteria_lines}\n\n" + f"Compute overall score as the weighted average: " + f"sum(score * weight / 100) for each criterion." + ) + return [ + {"role": "system", "content": system_prompt or _JUDGE_SYSTEM_PROMPT}, + {"role": "user", "content": user_content}, + ] + + +async def aevaluate( + response: Any, + criteria: List[EvalCriterion], + judge_model: str, + system_prompt: Optional[str] = None, + iteration: int = 0, + litellm_kwargs: Optional[Dict] = None, +) -> EvalResult: + """ + Async SDK primitive: run an LLM-judge eval on a response. + + Args: + response: ModelResponse or plain string to evaluate. + criteria: List of EvalCriterion dicts (name, weight, description). + judge_model: Model name to use as judge (e.g. "claude-sonnet-4-6"). + system_prompt: Override the default judge system prompt. + iteration: Which retry iteration this is (0-indexed). Used for logging. + litellm_kwargs: Extra kwargs forwarded to litellm.acompletion (e.g. api_base, api_key). + + Returns: + EvalResult — always returns (never raises). Check .eval_error for judge failures. + """ + import litellm + + start_time = datetime.now(timezone.utc) + response_text = _extract_response_text(response) + messages = _build_judge_messages(response_text, criteria, system_prompt) + + try: + judge_response = await litellm.acompletion( + model=judge_model, + messages=messages, + response_format={"type": "json_object"}, + **(litellm_kwargs or {}), + ) + raw_content = judge_response.choices[0].message.content or "{}" # type: ignore[union-attr] + raw_json: Dict = json.loads(raw_content) + except Exception as e: + # Fail open: judge error never blocks the response + return EvalResult( + overall_score=100.0, + passed=True, + judge_model=judge_model, + iteration=iteration, + eval_error=f"Judge call failed: {str(e)}", + ) + + try: + scores_raw = raw_json.get("scores", []) + verdicts: List[EvalVerdict] = [] + + # Build a weight map for computing overall score + weight_map: Dict[str, int] = { + c.get("name", ""): c.get("weight", 0) for c in criteria + } + total_weight = sum(weight_map.values()) or 1 + + weighted_sum = 0.0 + for s in scores_raw: + cname = s.get("criterion", "") + score = float(s.get("score", 0)) + weight = weight_map.get(cname, 0) + weighted_sum += score * (weight / total_weight) + verdicts.append( + EvalVerdict( + criterion_name=cname, + score=score, + reasoning=s.get("comment", ""), + passed=True, # per-criterion pass/fail evaluated by caller if threshold set + ) + ) + + overall_score = min(max(weighted_sum, 0.0), 100.0) + + end_time = datetime.now(timezone.utc) + duration = (end_time - start_time).total_seconds() + + return EvalResult( + overall_score=overall_score, + verdicts=verdicts, + passed=False, # caller sets this after threshold check + judge_model=judge_model, + iteration=iteration, + raw_judge_response=raw_json, + ) + + except Exception as e: + return EvalResult( + overall_score=100.0, + passed=True, + judge_model=judge_model, + iteration=iteration, + eval_error=f"Verdict parsing failed: {str(e)}", + ) + + +def evaluate( + response: Any, + criteria: List[EvalCriterion], + judge_model: str, + system_prompt: Optional[str] = None, + iteration: int = 0, + litellm_kwargs: Optional[Dict] = None, +) -> EvalResult: + """Sync wrapper for aevaluate(). See aevaluate() for full docs.""" + return asyncio.run( + aevaluate( + response=response, + criteria=criteria, + judge_model=judge_model, + system_prompt=system_prompt, + iteration=iteration, + litellm_kwargs=litellm_kwargs, + ) + )