scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-24 00:52:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 90

fix(eval_engine): use llm_router.acompletion for retry, not raw litellm

Browse source
This commit is contained in:
Ishaan Jaffer 2026-04-22 19:55:46 -07:00
parent 7bae837c17
commit a9f699c683
No known key found for this signature in database
1 changed files with 35 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
litellm/proxy/eval_management/eval_engine.py
View file

@ -89,6 +89,37 @@ async def run_evals_for_agent(
"judge_model": judge_model,
"iteration": eval_result.iteration,
"eval_error": eval_result.eval_error,
"verdicts": [
{
"criterion_name": (
v.get("criterion_name")
if isinstance(v, dict)
else getattr(v, "criterion_name", "")
),
"score": (
v.get("score")
if isinstance(v, dict)
else getattr(v, "score", 0)
),
"reasoning": (
v.get("reasoning")
if isinstance(v, dict)
else getattr(v, "reasoning", "")
),
"passed": (
v.get("passed")
if isinstance(v, dict)
else getattr(v, "passed", True)
),
"weight": (
v.get("weight")
if isinstance(v, dict)
else getattr(v, "weight", None)
),
}
for v in (eval_result.verdicts or [])
],
"threshold": threshold,
}
eval_information.append(eval_info)
@ -171,13 +202,13 @@ async def _run_single_eval_with_retry(
if messages and feedback_msg:
data["messages"] = messages + [{"role": "user", "content": feedback_msg}]
# Re-invoke the agent's LLM with the feedback
# Re-invoke the agent's LLM via the proxy router so routing/auth policies apply
try:
import litellm
from litellm.proxy.proxy_server import llm_router
model = data.get("model", "")
if model:
retry_response = await litellm.acompletion(
if model and llm_router is not None:
retry_response = await llm_router.acompletion(
model=model,
messages=data["messages"],
)