add EvalFailedError exception with status_code 422

litellm/exceptions.py

@@ -9,7 +9,7 @@
 
 ## LiteLLM versions of the OpenAI Exception Types
 
-from typing import Optional
+from typing import Any, Optional
 
 import httpx
 import openai
@@ -502,6 +502,25 @@ class ContentPolicyViolationError(BadRequestError):  # type: ignore
         return _message
 
 
+class EvalFailedError(Exception):
+    """Raised when an LLM-judge eval fails to meet the acceptance threshold."""
+
+    status_code: int = 422
+
+    def __init__(
+        self,
+        message: str,
+        eval_result: Any,
+        model: str = "",
+        llm_provider: str = "",
+    ):
+        self.eval_result = eval_result
+        self.model = model
+        self.llm_provider = llm_provider
+        self.status_code = 422
+        super().__init__(message)
+
+
 class ServiceUnavailableError(openai.APIStatusError):  # type: ignore
     def __init__(
         self,