Add eval run endpoints

litellm/llms/openai/evals/transformation.py

@@ -13,11 +13,17 @@ from litellm.llms.base_llm.evals.transformation import (
 )
 from litellm.types.llms.openai_evals import (
     CancelEvalResponse,
+    CancelRunResponse,
     CreateEvalRequest,
+    CreateRunRequest,
     DeleteEvalResponse,
     Eval,
     ListEvalsParams,
     ListEvalsResponse,
+    ListRunsParams,
+    ListRunsResponse,
+    Run,
+    RunDeleteResponse,
     UpdateEvalRequest,
 )
 from litellm.types.router import GenericLiteLLMParams
@@ -256,3 +262,165 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
         verbose_logger.debug("Transforming cancel eval response: %s", response_json)
 
         return CancelEvalResponse(**response_json)
+
+    # Run API Transformations
+    def transform_create_run_request(
+        self,
+        eval_id: str,
+        create_request: CreateRunRequest,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict]:
+        """Transform create run request for OpenAI"""
+        api_base = "https://api.openai.com"
+        if litellm_params and litellm_params.api_base:
+            api_base = litellm_params.api_base
+
+        url = f"{api_base}/v1/evals/{eval_id}/runs"
+
+        # Build request body
+        request_body = {k: v for k, v in create_request.items() if v is not None}
+
+        verbose_logger.debug(
+            "Create run request - URL: %s, body: %s", url, request_body
+        )
+
+        return url, request_body
+
+    def transform_create_run_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> Run:
+        """Transform OpenAI response to Run object"""
+        response_json = raw_response.json()
+        verbose_logger.debug("Transforming create run response: %s", response_json)
+
+        return Run(**response_json)
+
+    def transform_list_runs_request(
+        self,
+        eval_id: str,
+        list_params: ListRunsParams,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict]:
+        """Transform list runs request for OpenAI"""
+        api_base = "https://api.openai.com"
+        if litellm_params and litellm_params.api_base:
+            api_base = litellm_params.api_base
+
+        url = f"{api_base}/v1/evals/{eval_id}/runs"
+
+        # Build query parameters
+        query_params: Dict[str, Any] = {}
+        if "limit" in list_params and list_params["limit"]:
+            query_params["limit"] = list_params["limit"]
+        if "after" in list_params and list_params["after"]:
+            query_params["after"] = list_params["after"]
+        if "before" in list_params and list_params["before"]:
+            query_params["before"] = list_params["before"]
+        if "order" in list_params and list_params["order"]:
+            query_params["order"] = list_params["order"]
+
+        verbose_logger.debug(
+            "List runs request made to OpenAI Evals endpoint with params: %s",
+            query_params,
+        )
+
+        return url, query_params
+
+    def transform_list_runs_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> ListRunsResponse:
+        """Transform OpenAI response to ListRunsResponse"""
+        response_json = raw_response.json()
+        verbose_logger.debug("Transforming list runs response: %s", response_json)
+
+        return ListRunsResponse(**response_json)
+
+    def transform_get_run_request(
+        self,
+        eval_id: str,
+        run_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict]:
+        """Transform get run request for OpenAI"""
+        url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
+
+        verbose_logger.debug("Get run request - URL: %s", url)
+
+        return url, headers
+
+    def transform_get_run_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> Run:
+        """Transform OpenAI response to Run object"""
+        response_json = raw_response.json()
+        verbose_logger.debug("Transforming get run response: %s", response_json)
+
+        return Run(**response_json)
+
+    def transform_cancel_run_request(
+        self,
+        eval_id: str,
+        run_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict, Dict]:
+        """Transform cancel run request for OpenAI"""
+        url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}/cancel"
+
+        # Empty body for cancel request
+        request_body: Dict[str, Any] = {}
+
+        verbose_logger.debug("Cancel run request - URL: %s", url)
+
+        return url, headers, request_body
+
+    def transform_cancel_run_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> CancelRunResponse:
+        """Transform OpenAI response to CancelRunResponse"""
+        response_json = raw_response.json()
+        verbose_logger.debug("Transforming cancel run response: %s", response_json)
+
+        return CancelRunResponse(**response_json)
+
+    def transform_delete_run_request(
+        self,
+        eval_id: str,
+        run_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict, Dict]:
+        """Transform delete run request for OpenAI"""
+        url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
+
+        # Empty body for delete request
+        request_body: Dict[str, Any] = {}
+
+        verbose_logger.debug("Delete run request - URL: %s", url)
+
+        return url, headers, request_body
+
+    def transform_delete_run_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> RunDeleteResponse:
+        """Transform OpenAI response to RunDeleteResponse"""
+        response_json = raw_response.json()
+        verbose_logger.debug("Transforming delete run response: %s", response_json)
+
+        return RunDeleteResponse(**response_json)

litellm/proxy/openai_evals_endpoints/endpoints.py

@@ -12,9 +12,13 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
 from litellm.types.llms.openai_evals import (
     CancelEvalResponse,
+    CancelRunResponse,
     DeleteEvalResponse,
     Eval,
     ListEvalsResponse,
+    ListRunsResponse,
+    Run,
+    RunDeleteResponse,
 )
 
 router = APIRouter()
@@ -588,3 +592,476 @@ async def cancel_eval(
             proxy_logging_obj=proxy_logging_obj,
             version=version,
         )
+
+# ===================================
+# Run API Endpoints
+# ===================================
+
+
+@router.post(
+    "/v1/evals/{eval_id}/runs",
+    tags=["OpenAI Evals API - Runs"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=Run,
+)
+async def create_run(
+    eval_id: str,
+    fastapi_response: Response,
+    request: Request,
+    custom_llm_provider: Optional[str] = "openai",
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    Create a new run for an evaluation.
+
+    Model-based routing (for multi-account support):
+    - Pass model via header: `x-litellm-model: gpt-4-account-1`
+    - Pass model via query: `?model=gpt-4-account-1`
+    - Pass model via body: `{"model": "gpt-4-account-1"}`
+    - Pass model via completion.model: `{"completion": {"model": "gpt-4-account-1"}}`
+
+    Example usage:
+    ```bash
+    curl -X POST "http://localhost:4000/v1/evals/eval_123/runs" \
+      -H "Authorization: Bearer your-key" \
+      -H "Content-Type: application/json" \
+      -d '{
+        "data_source": {"type": "dataset", "dataset_id": "dataset_123"},
+        "completion": {"model": "gpt-4", "temperature": 0.7}
+      }'
+    ```
+
+    Returns: Run object with id, status, timestamps, etc.
+    """
+    from litellm.proxy.proxy_server import (
+        general_settings,
+        llm_router,
+        proxy_config,
+        proxy_logging_obj,
+        select_data_generator,
+        user_api_base,
+        user_max_tokens,
+        user_model,
+        user_request_timeout,
+        user_temperature,
+        version,
+    )
+
+    # Read request body
+    body = await request.body()
+    data = orjson.loads(body) if body else {}
+
+    # Set eval_id from path parameter
+    data["eval_id"] = eval_id
+
+    # Extract model for routing (header > query > body > completion.model)
+    model = (
+        request.headers.get("x-litellm-model")
+        or request.query_params.get("model")
+        or data.get("model")
+        or (data.get("completion", {}).get("model") if isinstance(data.get("completion"), dict) else None)
+    )
+    if model:
+        data["model"] = model
+
+    if "custom_llm_provider" not in data:
+        data["custom_llm_provider"] = custom_llm_provider
+
+    # Process request using ProxyBaseLLMRequestProcessing
+    processor = ProxyBaseLLMRequestProcessing(data=data)
+    try:
+        return await processor.base_process_llm_request(
+            request=request,
+            fastapi_response=fastapi_response,
+            user_api_key_dict=user_api_key_dict,
+            route_type="acreate_run",
+            proxy_logging_obj=proxy_logging_obj,
+            llm_router=llm_router,
+            general_settings=general_settings,
+            proxy_config=proxy_config,
+            select_data_generator=select_data_generator,
+            model=data.get("model"),
+            user_model=user_model,
+            user_temperature=user_temperature,
+            user_request_timeout=user_request_timeout,
+            user_max_tokens=user_max_tokens,
+            user_api_base=user_api_base,
+            version=version,
+        )
+    except Exception as e:
+        raise await processor._handle_llm_api_exception(
+            e=e,
+            user_api_key_dict=user_api_key_dict,
+            proxy_logging_obj=proxy_logging_obj,
+            version=version,
+        )
+
+
+@router.get(
+    "/v1/evals/{eval_id}/runs",
+    tags=["OpenAI Evals API - Runs"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=ListRunsResponse,
+)
+async def list_runs(
+    eval_id: str,
+    fastapi_response: Response,
+    request: Request,
+    limit: Optional[int] = 20,
+    after: Optional[str] = None,
+    before: Optional[str] = None,
+    order: Optional[str] = None,
+    custom_llm_provider: Optional[str] = "openai",
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    List all runs for an evaluation with pagination.
+
+    Model-based routing (for multi-account support):
+    - Pass model via header: `x-litellm-model: gpt-4-account-1`
+    - Pass model via query: `?model=gpt-4-account-1`
+
+    Example usage:
+    ```bash
+    curl "http://localhost:4000/v1/evals/eval_123/runs?limit=10" \
+      -H "Authorization: Bearer your-key"
+    ```
+
+    Returns: ListRunsResponse with list of runs
+    """
+    from litellm.proxy.proxy_server import (
+        general_settings,
+        llm_router,
+        proxy_config,
+        proxy_logging_obj,
+        select_data_generator,
+        user_api_base,
+        user_max_tokens,
+        user_model,
+        user_request_timeout,
+        user_temperature,
+        version,
+    )
+
+    # Build request data
+    data = {
+        "eval_id": eval_id,
+        "limit": limit,
+        "after": after,
+        "before": before,
+        "order": order,
+    }
+
+    # Extract model for routing (header > query)
+    model = request.headers.get("x-litellm-model") or request.query_params.get(
+        "model"
+    )
+    if model:
+        data["model"] = model
+
+    if "custom_llm_provider" not in data:
+        data["custom_llm_provider"] = custom_llm_provider
+
+    # Process request using ProxyBaseLLMRequestProcessing
+    processor = ProxyBaseLLMRequestProcessing(data=data)
+    try:
+        return await processor.base_process_llm_request(
+            request=request,
+            fastapi_response=fastapi_response,
+            user_api_key_dict=user_api_key_dict,
+            route_type="alist_runs",
+            proxy_logging_obj=proxy_logging_obj,
+            llm_router=llm_router,
+            general_settings=general_settings,
+            proxy_config=proxy_config,
+            select_data_generator=select_data_generator,
+            model=data.get("model"),
+            user_model=user_model,
+            user_temperature=user_temperature,
+            user_request_timeout=user_request_timeout,
+            user_max_tokens=user_max_tokens,
+            user_api_base=user_api_base,
+            version=version,
+        )
+    except Exception as e:
+        raise await processor._handle_llm_api_exception(
+            e=e,
+            user_api_key_dict=user_api_key_dict,
+            proxy_logging_obj=proxy_logging_obj,
+            version=version,
+        )
+
+
+@router.get(
+    "/v1/evals/{eval_id}/runs/{run_id}",
+    tags=["OpenAI Evals API - Runs"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=Run,
+)
+async def get_run(
+    eval_id: str,
+    run_id: str,
+    fastapi_response: Response,
+    request: Request,
+    custom_llm_provider: Optional[str] = "openai",
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    Get a specific run by ID.
+
+    Model-based routing (for multi-account support):
+    - Pass model via header: `x-litellm-model: gpt-4-account-1`
+    - Pass model via query: `?model=gpt-4-account-1`
+
+    Example usage:
+    ```bash
+    curl "http://localhost:4000/v1/evals/eval_123/runs/run_456" \
+      -H "Authorization: Bearer your-key"
+    ```
+
+    Returns: Run object with full details
+    """
+    from litellm.proxy.proxy_server import (
+        general_settings,
+        llm_router,
+        proxy_config,
+        proxy_logging_obj,
+        select_data_generator,
+        user_api_base,
+        user_max_tokens,
+        user_model,
+        user_request_timeout,
+        user_temperature,
+        version,
+    )
+
+    # Build request data
+    data = {
+        "eval_id": eval_id,
+        "run_id": run_id,
+    }
+
+    # Extract model for routing (header > query)
+    model = request.headers.get("x-litellm-model") or request.query_params.get(
+        "model"
+    )
+    if model:
+        data["model"] = model
+
+    if "custom_llm_provider" not in data:
+        data["custom_llm_provider"] = custom_llm_provider
+
+    # Process request using ProxyBaseLLMRequestProcessing
+    processor = ProxyBaseLLMRequestProcessing(data=data)
+    try:
+        return await processor.base_process_llm_request(
+            request=request,
+            fastapi_response=fastapi_response,
+            user_api_key_dict=user_api_key_dict,
+            route_type="aget_run",
+            proxy_logging_obj=proxy_logging_obj,
+            llm_router=llm_router,
+            general_settings=general_settings,
+            proxy_config=proxy_config,
+            select_data_generator=select_data_generator,
+            model=data.get("model"),
+            user_model=user_model,
+            user_temperature=user_temperature,
+            user_request_timeout=user_request_timeout,
+            user_max_tokens=user_max_tokens,
+            user_api_base=user_api_base,
+            version=version,
+        )
+    except Exception as e:
+        raise await processor._handle_llm_api_exception(
+            e=e,
+            user_api_key_dict=user_api_key_dict,
+            proxy_logging_obj=proxy_logging_obj,
+            version=version,
+        )
+
+
+@router.post(
+    "/v1/evals/{eval_id}/runs/{run_id}",
+    tags=["OpenAI Evals API - Runs"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=CancelRunResponse,
+)
+async def cancel_run(
+    eval_id: str,
+    run_id: str,
+    fastapi_response: Response,
+    request: Request,
+    custom_llm_provider: Optional[str] = "openai",
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    Cancel a running run.
+
+    Model-based routing (for multi-account support):
+    - Pass model via header: `x-litellm-model: gpt-4-account-1`
+    - Pass model via query: `?model=gpt-4-account-1`
+
+    Example usage:
+    ```bash
+    curl -X POST "http://localhost:4000/v1/evals/eval_123/runs/run_456/cancel" \
+      -H "Authorization: Bearer your-key"
+    ```
+
+    Returns: CancelRunResponse with cancellation confirmation
+    """
+    from litellm.proxy.proxy_server import (
+        general_settings,
+        llm_router,
+        proxy_config,
+        proxy_logging_obj,
+        select_data_generator,
+        user_api_base,
+        user_max_tokens,
+        user_model,
+        user_request_timeout,
+        user_temperature,
+        version,
+    )
+
+    # Read request body (optional for cancel)
+    body = await request.body()
+    data = orjson.loads(body) if body else {}
+
+    # Set eval_id and run_id from path parameters
+    data["eval_id"] = eval_id
+    data["run_id"] = run_id
+
+    # Extract model for routing (header > query > body)
+    model = (
+        data.get("model")
+        or request.query_params.get("model")
+        or request.headers.get("x-litellm-model")
+    )
+    if model:
+        data["model"] = model
+
+    if "custom_llm_provider" not in data:
+        data["custom_llm_provider"] = custom_llm_provider
+
+    # Process request using ProxyBaseLLMRequestProcessing
+    processor = ProxyBaseLLMRequestProcessing(data=data)
+    try:
+        return await processor.base_process_llm_request(
+            request=request,
+            fastapi_response=fastapi_response,
+            user_api_key_dict=user_api_key_dict,
+            route_type="acancel_run",
+            proxy_logging_obj=proxy_logging_obj,
+            llm_router=llm_router,
+            general_settings=general_settings,
+            proxy_config=proxy_config,
+            select_data_generator=select_data_generator,
+            model=data.get("model"),
+            user_model=user_model,
+            user_temperature=user_temperature,
+            user_request_timeout=user_request_timeout,
+            user_max_tokens=user_max_tokens,
+            user_api_base=user_api_base,
+            version=version,
+        )
+    except Exception as e:
+        raise await processor._handle_llm_api_exception(
+            e=e,
+            user_api_key_dict=user_api_key_dict,
+            proxy_logging_obj=proxy_logging_obj,
+            version=version,
+        )
+
+
+@router.delete(
+    "/v1/evals/{eval_id}/runs/{run_id}",
+    tags=["OpenAI Evals API - Runs"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=RunDeleteResponse,
+)
+async def delete_run(
+    eval_id: str,
+    run_id: str,
+    fastapi_response: Response,
+    request: Request,
+    custom_llm_provider: Optional[str] = "openai",
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    Delete a run.
+
+    Model-based routing (for multi-account support):
+    - Pass model via header: `x-litellm-model: gpt-4-account-1`
+    - Pass model via query: `?model=gpt-4-account-1`
+
+    Example usage:
+    ```bash
+    curl -X DELETE "http://localhost:4000/v1/evals/eval_123/runs/run_456" \
+      -H "Authorization: Bearer your-key"
+    ```
+
+    Returns: RunDeleteResponse with deletion confirmation
+    """
+    from litellm.proxy.proxy_server import (
+        general_settings,
+        llm_router,
+        proxy_config,
+        proxy_logging_obj,
+        select_data_generator,
+        user_api_base,
+        user_max_tokens,
+        user_model,
+        user_request_timeout,
+        user_temperature,
+        version,
+    )
+
+    # Read request body (optional for delete)
+    body = await request.body()
+    data = orjson.loads(body) if body else {}
+
+    # Set eval_id and run_id from path parameters
+    data["eval_id"] = eval_id
+    data["run_id"] = run_id
+
+    # Extract model for routing (header > query > body)
+    model = (
+        data.get("model")
+        or request.query_params.get("model")
+        or request.headers.get("x-litellm-model")
+    )
+    if model:
+        data["model"] = model
+
+    if "custom_llm_provider" not in data:
+        data["custom_llm_provider"] = custom_llm_provider
+
+    # Process request using ProxyBaseLLMRequestProcessing
+    processor = ProxyBaseLLMRequestProcessing(data=data)
+    try:
+        return await processor.base_process_llm_request(
+            request=request,
+            fastapi_response=fastapi_response,
+            user_api_key_dict=user_api_key_dict,
+            route_type="adelete_run",
+            proxy_logging_obj=proxy_logging_obj,
+            llm_router=llm_router,
+            general_settings=general_settings,
+            proxy_config=proxy_config,
+            select_data_generator=select_data_generator,
+            model=data.get("model"),
+            user_model=user_model,
+            user_temperature=user_temperature,
+            user_request_timeout=user_request_timeout,
+            user_max_tokens=user_max_tokens,
+            user_api_base=user_api_base,
+            version=version,
+        )
+    except Exception as e:
+        raise await processor._handle_llm_api_exception(
+            e=e,
+            user_api_key_dict=user_api_key_dict,
+            proxy_logging_obj=proxy_logging_obj,
+            version=version,
+        )