diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 7f8832da005..76220ed3c67 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -326,6 +326,50 @@ def _enrich_http_exception_with_guardrail_context( detail.setdefault("guardrail_mode", event_hook) +async def _patch_eval_spend_log( + prisma_client: Any, + request_id: str, + eval_info: Any, + _retried: bool = False, +) -> None: + """Patch eval_information into an existing spend log row. + + The DB-logger callback that writes spend logs runs concurrently with + async_post_call_success_hook. If the row isn't committed yet, we sleep + 500 ms and retry once — covering the common timing gap without busy-looping. + """ + import asyncio + import json as _json + + try: + existing = await prisma_client.db.litellm_spendlogs.find_unique( + where={"request_id": request_id} + ) + if existing is None: + if not _retried: + await asyncio.sleep(0.5) + await _patch_eval_spend_log( + prisma_client, request_id, eval_info, _retried=True + ) + return + verbose_proxy_logger.warning( + f"[Evals] Spend log row {request_id} not found after retry; eval_information not persisted" + ) + return + meta = existing.metadata or {} + if isinstance(meta, str): + meta = _json.loads(meta) + meta["eval_information"] = eval_info + await prisma_client.db.litellm_spendlogs.update( + where={"request_id": request_id}, + data={"metadata": _json.dumps(meta)}, + ) + except Exception as _e: + verbose_proxy_logger.warning( + f"[Evals] Failed to patch spend log {request_id}: {_e}" + ) + + class ProxyLogging: """ Logging/Custom Handlers for proxy. @@ -2092,6 +2136,8 @@ class ProxyLogging: ############ Eval Gate [Beta] ############################## ############################################################ + from litellm.proxy.proxy_server import prisma_client as _prisma_client + try: _agent_id = (data.get("metadata") or {}).get("agent_id") or data.get( "agent_id" @@ -2100,9 +2146,6 @@ class ProxyLogging: from litellm.proxy.eval_management.eval_engine import ( run_evals_for_agent, ) - from litellm.proxy.proxy_server import ( - prisma_client as _prisma_client, - ) if _prisma_client is not None: response = await run_evals_for_agent( @@ -2112,10 +2155,37 @@ class ProxyLogging: user_api_key_dict=user_api_key_dict, prisma_client=_prisma_client, ) + # Patch the spend log — eval runs after the spend logger fires, + # so we update the row directly to make eval_information visible in Logs UI. + # For A2A calls, the spend log request_id is the JSON-RPC id (data["id"]), + # not the litellm_call_id UUID generated by common_request_processing. + _eval_info = (data.get("metadata") or {}).get("eval_information") + # For regular completions the spend log uses response.id (chatcmpl-XXX). + # For A2A calls it uses the JSON-RPC id stored in data["id"]. + _request_id = ( + getattr(response, "id", None) + or data.get("id") + or data.get("litellm_call_id") + ) + if _eval_info and _request_id: + await _patch_eval_spend_log( + _prisma_client, _request_id, _eval_info + ) except Exception as e: from fastapi import HTTPException if isinstance(e, HTTPException): + # Even on block, patch the spend log so Logs UI shows the failed eval scores + _eval_info = (data.get("metadata") or {}).get("eval_information") + _request_id = ( + getattr(response, "id", None) + or data.get("id") + or data.get("litellm_call_id") + ) + if _eval_info and _request_id and _prisma_client is not None: + await _patch_eval_spend_log( + _prisma_client, _request_id, _eval_info + ) raise verbose_proxy_logger.warning( f"[Evals] Unexpected error in eval gate: {e}"