diff --git a/ui/litellm-dashboard/src/components/agents/agent_info.tsx b/ui/litellm-dashboard/src/components/agents/agent_info.tsx index d543be8356a..846b12cd505 100644 --- a/ui/litellm-dashboard/src/components/agents/agent_info.tsx +++ b/ui/litellm-dashboard/src/components/agents/agent_info.tsx @@ -1,9 +1,9 @@ import React, { useState, useEffect } from "react"; import { Card, Title, Text, Button as TremorButton, Tab, TabGroup, TabList, TabPanel, TabPanels} from "@tremor/react"; -import { Form, Input, InputNumber, Button as AntButton, Spin, Descriptions, Divider } from "antd"; +import { Form, Input, InputNumber, Button as AntButton, Spin, Descriptions, Divider, Modal, Tag, Space, Table, Popconfirm, message, Typography, Select as AntSelect } from "antd"; import MessageManager from "@/components/molecules/message_manager"; import { ArrowLeftIcon } from "@heroicons/react/outline"; -import { getAgentInfo, patchAgentCall, getAgentCreateMetadata, AgentCreateInfo } from "../networking"; +import { getAgentInfo, patchAgentCall, getAgentCreateMetadata, AgentCreateInfo, getAgentEvals, listLiteLLMEvals, attachEvalToAgent, detachEvalFromAgent } from "../networking"; import { Agent } from "./types"; import AgentFormFields from "./agent_form_fields"; import DynamicAgentFormFields, { buildDynamicAgentData } from "./dynamic_agent_form_fields"; @@ -11,6 +11,177 @@ import { buildAgentDataFromForm, parseAgentForForm } from "./agent_config"; import AgentCostView from "./agent_cost_view"; import { detectAgentType, parseDynamicAgentForForm } from "./agent_type_utils"; +function AgentEvalsTab({ agentId, accessToken }: { agentId: string; accessToken: string | null }) { + const [attachedEvals, setAttachedEvals] = React.useState([]); + const [allEvals, setAllEvals] = React.useState([]); + const [loading, setLoading] = React.useState(false); + const [modalOpen, setModalOpen] = React.useState(false); + const [selectedEvalId, setSelectedEvalId] = React.useState(); + const [onFailure, setOnFailure] = React.useState<"block" | "log">("block"); + const [attaching, setAttaching] = React.useState(false); + + const fetchData = React.useCallback(async () => { + if (!accessToken || !agentId) return; + setLoading(true); + try { + const [attached, catalog] = await Promise.all([ + getAgentEvals(accessToken, agentId), + listLiteLLMEvals(accessToken), + ]); + setAttachedEvals(attached); + setAllEvals(catalog); + } catch (e: any) { + message.error(`Failed to load evals: ${e.message}`); + } finally { + setLoading(false); + } + }, [accessToken, agentId]); + + React.useEffect(() => { + fetchData(); + }, [fetchData]); + + const handleAttach = async () => { + if (!accessToken || !selectedEvalId) return; + setAttaching(true); + try { + await attachEvalToAgent(accessToken, agentId, { eval_id: selectedEvalId, on_failure: onFailure }); + message.success("Eval attached"); + setModalOpen(false); + setSelectedEvalId(undefined); + fetchData(); + } catch (e: any) { + message.error(`Failed to attach eval: ${e.message}`); + } finally { + setAttaching(false); + } + }; + + const handleDetach = async (evalId: string) => { + if (!accessToken) return; + try { + await detachEvalFromAgent(accessToken, agentId, evalId); + message.success("Eval detached"); + fetchData(); + } catch (e: any) { + message.error(`Failed to detach eval: ${e.message}`); + } + }; + + const attachedEvalIds = new Set(attachedEvals.map((e: any) => e.eval_id)); + const availableToAttach = allEvals.filter((e: any) => !attachedEvalIds.has(e.eval_id)); + + const evalColumns = [ + { + title: "Eval Name", + dataIndex: "eval_name", + key: "eval_name", + render: (v: string) => {v}, + }, + { + title: "On Failure", + dataIndex: "on_failure", + key: "on_failure", + render: (v: string) => ( + {v === "block" ? "Block" : "Log only"} + ), + }, + { + title: "Threshold", + key: "threshold", + render: (_: any, r: any) => { + const t = r.overall_threshold_override ?? r.eval?.overall_threshold; + return t != null ? ≥ {t} : default; + }, + }, + { + title: "Judge Model", + key: "judge_model", + render: (_: any, r: any) => r.eval?.judge_model || "—", + }, + { + title: "Actions", + key: "actions", + render: (_: any, r: any) => ( + handleDetach(r.eval_id)} + okText="Remove" + okButtonProps={{ danger: true }} + > + Remove + + ), + }, + ]; + + return ( +
+
+ + Evals gate this agent's responses. Hard-gate (Block) returns 422 on failure; Soft-observer (Log) records the result without blocking. + + setModalOpen(true)} + disabled={availableToAttach.length === 0} + > + Attach Eval + +
+ + + Attach Eval Beta + + } + open={modalOpen} + onCancel={() => { setModalOpen(false); setSelectedEvalId(undefined); }} + onOk={handleAttach} + confirmLoading={attaching} + okText="Attach" + > +
+ + setSelectedEvalId(val)} + style={{ width: "100%" }} + options={availableToAttach.map((e: any) => ({ label: e.eval_name, value: e.eval_id }))} + /> + + + setOnFailure(val)} + style={{ width: "100%" }} + options={[ + { label: "Block (return 422)", value: "block" }, + { label: "Log only (observe)", value: "log" }, + ]} + /> + + +
+ + ); +} + interface AgentInfoViewProps { agentId: string; onClose: () => void; @@ -165,6 +336,7 @@ const AgentInfoView: React.FC = ({ Overview {isAdmin ? Settings : <>} + Evals Beta @@ -337,6 +509,10 @@ const AgentInfoView: React.FC = ({ )} + {/* Evals Panel */} + + +