diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py index 975bf172f1c..c24dbf8637a 100644 --- a/litellm/llms/openai/evals/transformation.py +++ b/litellm/llms/openai/evals/transformation.py @@ -13,11 +13,17 @@ from litellm.llms.base_llm.evals.transformation import ( ) from litellm.types.llms.openai_evals import ( CancelEvalResponse, + CancelRunResponse, CreateEvalRequest, + CreateRunRequest, DeleteEvalResponse, Eval, ListEvalsParams, ListEvalsResponse, + ListRunsParams, + ListRunsResponse, + Run, + RunDeleteResponse, UpdateEvalRequest, ) from litellm.types.router import GenericLiteLLMParams @@ -256,3 +262,165 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): verbose_logger.debug("Transforming cancel eval response: %s", response_json) return CancelEvalResponse(**response_json) + + # Run API Transformations + def transform_create_run_request( + self, + eval_id: str, + create_request: CreateRunRequest, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform create run request for OpenAI""" + api_base = "https://api.openai.com" + if litellm_params and litellm_params.api_base: + api_base = litellm_params.api_base + + url = f"{api_base}/v1/evals/{eval_id}/runs" + + # Build request body + request_body = {k: v for k, v in create_request.items() if v is not None} + + verbose_logger.debug( + "Create run request - URL: %s, body: %s", url, request_body + ) + + return url, request_body + + def transform_create_run_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> Run: + """Transform OpenAI response to Run object""" + response_json = raw_response.json() + verbose_logger.debug("Transforming create run response: %s", response_json) + + return Run(**response_json) + + def transform_list_runs_request( + self, + eval_id: str, + list_params: ListRunsParams, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform list runs request for OpenAI""" + api_base = "https://api.openai.com" + if litellm_params and litellm_params.api_base: + api_base = litellm_params.api_base + + url = f"{api_base}/v1/evals/{eval_id}/runs" + + # Build query parameters + query_params: Dict[str, Any] = {} + if "limit" in list_params and list_params["limit"]: + query_params["limit"] = list_params["limit"] + if "after" in list_params and list_params["after"]: + query_params["after"] = list_params["after"] + if "before" in list_params and list_params["before"]: + query_params["before"] = list_params["before"] + if "order" in list_params and list_params["order"]: + query_params["order"] = list_params["order"] + + verbose_logger.debug( + "List runs request made to OpenAI Evals endpoint with params: %s", + query_params, + ) + + return url, query_params + + def transform_list_runs_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ListRunsResponse: + """Transform OpenAI response to ListRunsResponse""" + response_json = raw_response.json() + verbose_logger.debug("Transforming list runs response: %s", response_json) + + return ListRunsResponse(**response_json) + + def transform_get_run_request( + self, + eval_id: str, + run_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform get run request for OpenAI""" + url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}" + + verbose_logger.debug("Get run request - URL: %s", url) + + return url, headers + + def transform_get_run_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> Run: + """Transform OpenAI response to Run object""" + response_json = raw_response.json() + verbose_logger.debug("Transforming get run response: %s", response_json) + + return Run(**response_json) + + def transform_cancel_run_request( + self, + eval_id: str, + run_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict, Dict]: + """Transform cancel run request for OpenAI""" + url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}/cancel" + + # Empty body for cancel request + request_body: Dict[str, Any] = {} + + verbose_logger.debug("Cancel run request - URL: %s", url) + + return url, headers, request_body + + def transform_cancel_run_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> CancelRunResponse: + """Transform OpenAI response to CancelRunResponse""" + response_json = raw_response.json() + verbose_logger.debug("Transforming cancel run response: %s", response_json) + + return CancelRunResponse(**response_json) + + def transform_delete_run_request( + self, + eval_id: str, + run_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict, Dict]: + """Transform delete run request for OpenAI""" + url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}" + + # Empty body for delete request + request_body: Dict[str, Any] = {} + + verbose_logger.debug("Delete run request - URL: %s", url) + + return url, headers, request_body + + def transform_delete_run_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> RunDeleteResponse: + """Transform OpenAI response to RunDeleteResponse""" + response_json = raw_response.json() + verbose_logger.debug("Transforming delete run response: %s", response_json) + + return RunDeleteResponse(**response_json) diff --git a/litellm/proxy/openai_evals_endpoints/endpoints.py b/litellm/proxy/openai_evals_endpoints/endpoints.py index 5b40c30a779..2f1bb7839f8 100644 --- a/litellm/proxy/openai_evals_endpoints/endpoints.py +++ b/litellm/proxy/openai_evals_endpoints/endpoints.py @@ -12,9 +12,13 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.types.llms.openai_evals import ( CancelEvalResponse, + CancelRunResponse, DeleteEvalResponse, Eval, ListEvalsResponse, + ListRunsResponse, + Run, + RunDeleteResponse, ) router = APIRouter() @@ -588,3 +592,476 @@ async def cancel_eval( proxy_logging_obj=proxy_logging_obj, version=version, ) + +# =================================== +# Run API Endpoints +# =================================== + + +@router.post( + "/v1/evals/{eval_id}/runs", + tags=["OpenAI Evals API - Runs"], + dependencies=[Depends(user_api_key_auth)], + response_model=Run, +) +async def create_run( + eval_id: str, + fastapi_response: Response, + request: Request, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Create a new run for an evaluation. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + - Pass model via body: `{"model": "gpt-4-account-1"}` + - Pass model via completion.model: `{"completion": {"model": "gpt-4-account-1"}}` + + Example usage: + ```bash + curl -X POST "http://localhost:4000/v1/evals/eval_123/runs" \ + -H "Authorization: Bearer your-key" \ + -H "Content-Type: application/json" \ + -d '{ + "data_source": {"type": "dataset", "dataset_id": "dataset_123"}, + "completion": {"model": "gpt-4", "temperature": 0.7} + }' + ``` + + Returns: Run object with id, status, timestamps, etc. + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Read request body + body = await request.body() + data = orjson.loads(body) if body else {} + + # Set eval_id from path parameter + data["eval_id"] = eval_id + + # Extract model for routing (header > query > body > completion.model) + model = ( + request.headers.get("x-litellm-model") + or request.query_params.get("model") + or data.get("model") + or (data.get("completion", {}).get("model") if isinstance(data.get("completion"), dict) else None) + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="acreate_run", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.get( + "/v1/evals/{eval_id}/runs", + tags=["OpenAI Evals API - Runs"], + dependencies=[Depends(user_api_key_auth)], + response_model=ListRunsResponse, +) +async def list_runs( + eval_id: str, + fastapi_response: Response, + request: Request, + limit: Optional[int] = 20, + after: Optional[str] = None, + before: Optional[str] = None, + order: Optional[str] = None, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + List all runs for an evaluation with pagination. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + + Example usage: + ```bash + curl "http://localhost:4000/v1/evals/eval_123/runs?limit=10" \ + -H "Authorization: Bearer your-key" + ``` + + Returns: ListRunsResponse with list of runs + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Build request data + data = { + "eval_id": eval_id, + "limit": limit, + "after": after, + "before": before, + "order": order, + } + + # Extract model for routing (header > query) + model = request.headers.get("x-litellm-model") or request.query_params.get( + "model" + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="alist_runs", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.get( + "/v1/evals/{eval_id}/runs/{run_id}", + tags=["OpenAI Evals API - Runs"], + dependencies=[Depends(user_api_key_auth)], + response_model=Run, +) +async def get_run( + eval_id: str, + run_id: str, + fastapi_response: Response, + request: Request, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get a specific run by ID. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + + Example usage: + ```bash + curl "http://localhost:4000/v1/evals/eval_123/runs/run_456" \ + -H "Authorization: Bearer your-key" + ``` + + Returns: Run object with full details + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Build request data + data = { + "eval_id": eval_id, + "run_id": run_id, + } + + # Extract model for routing (header > query) + model = request.headers.get("x-litellm-model") or request.query_params.get( + "model" + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="aget_run", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.post( + "/v1/evals/{eval_id}/runs/{run_id}", + tags=["OpenAI Evals API - Runs"], + dependencies=[Depends(user_api_key_auth)], + response_model=CancelRunResponse, +) +async def cancel_run( + eval_id: str, + run_id: str, + fastapi_response: Response, + request: Request, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Cancel a running run. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + + Example usage: + ```bash + curl -X POST "http://localhost:4000/v1/evals/eval_123/runs/run_456/cancel" \ + -H "Authorization: Bearer your-key" + ``` + + Returns: CancelRunResponse with cancellation confirmation + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Read request body (optional for cancel) + body = await request.body() + data = orjson.loads(body) if body else {} + + # Set eval_id and run_id from path parameters + data["eval_id"] = eval_id + data["run_id"] = run_id + + # Extract model for routing (header > query > body) + model = ( + data.get("model") + or request.query_params.get("model") + or request.headers.get("x-litellm-model") + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="acancel_run", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.delete( + "/v1/evals/{eval_id}/runs/{run_id}", + tags=["OpenAI Evals API - Runs"], + dependencies=[Depends(user_api_key_auth)], + response_model=RunDeleteResponse, +) +async def delete_run( + eval_id: str, + run_id: str, + fastapi_response: Response, + request: Request, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Delete a run. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + + Example usage: + ```bash + curl -X DELETE "http://localhost:4000/v1/evals/eval_123/runs/run_456" \ + -H "Authorization: Bearer your-key" + ``` + + Returns: RunDeleteResponse with deletion confirmation + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Read request body (optional for delete) + body = await request.body() + data = orjson.loads(body) if body else {} + + # Set eval_id and run_id from path parameters + data["eval_id"] = eval_id + data["run_id"] = run_id + + # Extract model for routing (header > query > body) + model = ( + data.get("model") + or request.query_params.get("model") + or request.headers.get("x-litellm-model") + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="adelete_run", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + )