mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
Add eval run endpoints and methods
This commit is contained in:
Sameer Kankute
parent
8b75979fdc
commit
b246c3c56c
5 changed files with 1455 additions and 1 deletions
|
|
@ -1734,7 +1734,9 @@ def __getattr__(name: str) -> Any:
|
|||
|
||||
# Lazy load evals module functions
|
||||
if name in ["acreate_eval", "alist_evals", "aget_eval", "aupdate_eval", "adelete_eval", "acancel_eval",
|
||||
"create_eval", "list_evals", "get_eval", "update_eval", "delete_eval", "cancel_eval"]:
|
||||
"create_eval", "list_evals", "get_eval", "update_eval", "delete_eval", "cancel_eval",
|
||||
"acreate_run", "alist_runs", "aget_run", "acancel_run", "adelete_run",
|
||||
"create_run", "list_runs", "get_run", "cancel_run", "delete_run"]:
|
||||
from litellm.evals.main import (
|
||||
acreate_eval,
|
||||
alist_evals,
|
||||
|
|
@ -1748,6 +1750,16 @@ def __getattr__(name: str) -> Any:
|
|||
update_eval,
|
||||
delete_eval,
|
||||
cancel_eval,
|
||||
acreate_run,
|
||||
alist_runs,
|
||||
aget_run,
|
||||
acancel_run,
|
||||
adelete_run,
|
||||
create_run,
|
||||
list_runs,
|
||||
get_run,
|
||||
cancel_run,
|
||||
delete_run,
|
||||
)
|
||||
return locals()[name]
|
||||
|
||||
|
|
|
|||
|
|
@ -17,11 +17,17 @@ from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig
|
|||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.types.llms.openai_evals import (
|
||||
CancelEvalResponse,
|
||||
CancelRunResponse,
|
||||
CreateEvalRequest,
|
||||
CreateRunRequest,
|
||||
DeleteEvalResponse,
|
||||
Eval,
|
||||
ListEvalsParams,
|
||||
ListEvalsResponse,
|
||||
ListRunsParams,
|
||||
ListRunsResponse,
|
||||
Run,
|
||||
RunDeleteResponse,
|
||||
UpdateEvalRequest,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
|
@ -1072,3 +1078,867 @@ def cancel_eval(
|
|||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
# ===================================
|
||||
# Run API Functions
|
||||
# ===================================
|
||||
|
||||
|
||||
@client
|
||||
async def acreate_run(
|
||||
eval_id: str,
|
||||
data_source: Dict[str, Any],
|
||||
name: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
extra_body: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Run:
|
||||
"""
|
||||
Async: Create a new run for an evaluation
|
||||
|
||||
Args:
|
||||
eval_id: The ID of the evaluation to run
|
||||
data_source: Data source configuration for the run (can be jsonl, completions, or responses type)
|
||||
name: Optional name for the run
|
||||
metadata: Optional additional metadata
|
||||
extra_headers: Additional headers for the request
|
||||
extra_query: Additional query parameters
|
||||
extra_body: Additional body parameters
|
||||
timeout: Request timeout
|
||||
custom_llm_provider: Provider name (e.g., 'openai')
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
Run object
|
||||
"""
|
||||
local_vars = locals()
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
kwargs["acreate_run"] = True
|
||||
|
||||
func = partial(
|
||||
create_run,
|
||||
eval_id=eval_id,
|
||||
data_source=data_source,
|
||||
name=name,
|
||||
metadata=metadata,
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
ctx = contextvars.copy_context()
|
||||
func_with_context = partial(ctx.run, func)
|
||||
init_response = await loop.run_in_executor(None, func_with_context)
|
||||
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
else:
|
||||
response = init_response
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
def create_run(
|
||||
eval_id: str,
|
||||
data_source: Dict[str, Any],
|
||||
name: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
extra_body: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Union[Run, Coroutine[Any, Any, Run]]:
|
||||
"""
|
||||
Create a new run for an evaluation
|
||||
|
||||
Args:
|
||||
eval_id: The ID of the evaluation to run
|
||||
data_source: Data source configuration for the run (can be jsonl, completions, or responses type)
|
||||
name: Optional name for the run
|
||||
metadata: Optional additional metadata
|
||||
extra_headers: Additional headers for the request
|
||||
extra_query: Additional query parameters
|
||||
extra_body: Additional body parameters
|
||||
timeout: Request timeout (default 600s for long-running operations)
|
||||
custom_llm_provider: Provider name (e.g., 'openai')
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
Run object
|
||||
"""
|
||||
local_vars = locals()
|
||||
try:
|
||||
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
|
||||
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
|
||||
_is_async = kwargs.pop("acreate_run", False) is True
|
||||
|
||||
# Get LiteLLM parameters
|
||||
litellm_params = GenericLiteLLMParams(**kwargs)
|
||||
|
||||
# Determine provider
|
||||
if custom_llm_provider is None:
|
||||
custom_llm_provider = "openai"
|
||||
|
||||
# Get provider config
|
||||
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
|
||||
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
|
||||
provider=litellm.LlmProviders(custom_llm_provider),
|
||||
)
|
||||
)
|
||||
|
||||
if evals_api_provider_config is None:
|
||||
raise ValueError(
|
||||
f"CREATE run is not supported for {custom_llm_provider}"
|
||||
)
|
||||
|
||||
# Build create request
|
||||
create_request: CreateRunRequest = {
|
||||
"data_source": data_source, # type: ignore
|
||||
}
|
||||
if name is not None:
|
||||
create_request["name"] = name
|
||||
# if metadata is not None:
|
||||
# create_request["metadata"] = metadata
|
||||
|
||||
# Merge extra_body if provided
|
||||
if extra_body:
|
||||
create_request.update(extra_body) # type: ignore
|
||||
|
||||
# Validate environment and get headers
|
||||
headers = extra_headers or {}
|
||||
headers = evals_api_provider_config.validate_environment(
|
||||
headers=headers, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
# Transform request
|
||||
api_base = litellm_params.api_base or DEFAULT_OPENAI_API_BASE
|
||||
url, request_body = evals_api_provider_config.transform_create_run_request(
|
||||
eval_id=eval_id,
|
||||
create_request=create_request,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Pre-call logging
|
||||
litellm_logging_obj.update_environment_variables(
|
||||
model=None,
|
||||
optional_params=request_body,
|
||||
litellm_params={
|
||||
"litellm_call_id": litellm_call_id,
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
# Make HTTP request (default 600s timeout for long-running operations)
|
||||
response = base_llm_http_handler.create_run_handler( # type: ignore
|
||||
url=url,
|
||||
request_body=request_body,
|
||||
evals_api_provider_config=evals_api_provider_config,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
extra_headers=headers,
|
||||
timeout=timeout or httpx.Timeout(timeout=600.0, connect=5.0),
|
||||
_is_async=_is_async,
|
||||
client=kwargs.get("client"),
|
||||
shared_session=kwargs.get("shared_session"),
|
||||
)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
async def alist_runs(
|
||||
eval_id: str,
|
||||
limit: Optional[int] = None,
|
||||
after: Optional[str] = None,
|
||||
before: Optional[str] = None,
|
||||
order: Optional[str] = None,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> ListRunsResponse:
|
||||
"""
|
||||
Async: List all runs for an evaluation
|
||||
|
||||
Args:
|
||||
eval_id: The ID of the evaluation
|
||||
limit: Number of results to return per page (max 100, default 20)
|
||||
after: Cursor for pagination - returns runs after this ID
|
||||
before: Cursor for pagination - returns runs before this ID
|
||||
order: Sort order ('asc' or 'desc', default 'desc')
|
||||
extra_headers: Additional headers for the request
|
||||
extra_query: Additional query parameters
|
||||
timeout: Request timeout
|
||||
custom_llm_provider: Provider name (e.g., 'openai')
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
ListRunsResponse object
|
||||
"""
|
||||
local_vars = locals()
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
kwargs["alist_runs"] = True
|
||||
|
||||
func = partial(
|
||||
list_runs,
|
||||
eval_id=eval_id,
|
||||
limit=limit,
|
||||
after=after,
|
||||
before=before,
|
||||
order=order,
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
ctx = contextvars.copy_context()
|
||||
func_with_context = partial(ctx.run, func)
|
||||
init_response = await loop.run_in_executor(None, func_with_context)
|
||||
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
else:
|
||||
response = init_response
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
def list_runs(
|
||||
eval_id: str,
|
||||
limit: Optional[int] = None,
|
||||
after: Optional[str] = None,
|
||||
before: Optional[str] = None,
|
||||
order: Optional[str] = None,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Union[ListRunsResponse, Coroutine[Any, Any, ListRunsResponse]]:
|
||||
"""
|
||||
List all runs for an evaluation
|
||||
|
||||
Args:
|
||||
eval_id: The ID of the evaluation
|
||||
limit: Number of results to return per page (max 100, default 20)
|
||||
after: Cursor for pagination - returns runs after this ID
|
||||
before: Cursor for pagination - returns runs before this ID
|
||||
order: Sort order ('asc' or 'desc', default 'desc')
|
||||
extra_headers: Additional headers for the request
|
||||
extra_query: Additional query parameters
|
||||
timeout: Request timeout
|
||||
custom_llm_provider: Provider name (e.g., 'openai')
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
ListRunsResponse object
|
||||
"""
|
||||
local_vars = locals()
|
||||
try:
|
||||
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
|
||||
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
|
||||
_is_async = kwargs.pop("alist_runs", False) is True
|
||||
|
||||
# Get LiteLLM parameters
|
||||
litellm_params = GenericLiteLLMParams(**kwargs)
|
||||
|
||||
# Determine provider
|
||||
if custom_llm_provider is None:
|
||||
custom_llm_provider = "openai"
|
||||
|
||||
# Get provider config
|
||||
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
|
||||
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
|
||||
provider=litellm.LlmProviders(custom_llm_provider),
|
||||
)
|
||||
)
|
||||
|
||||
if evals_api_provider_config is None:
|
||||
raise ValueError(f"LIST runs is not supported for {custom_llm_provider}")
|
||||
|
||||
# Build list parameters
|
||||
list_params: ListRunsParams = {}
|
||||
if limit is not None:
|
||||
list_params["limit"] = limit
|
||||
if after is not None:
|
||||
list_params["after"] = after
|
||||
if before is not None:
|
||||
list_params["before"] = before
|
||||
if order is not None:
|
||||
list_params["order"] = order # type: ignore
|
||||
|
||||
# Merge extra_query if provided
|
||||
if extra_query:
|
||||
list_params.update(extra_query) # type: ignore
|
||||
|
||||
# Validate environment and get headers
|
||||
headers = extra_headers or {}
|
||||
headers = evals_api_provider_config.validate_environment(
|
||||
headers=headers, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
# Transform request
|
||||
url, query_params = evals_api_provider_config.transform_list_runs_request(
|
||||
eval_id=eval_id,
|
||||
list_params=list_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Pre-call logging
|
||||
litellm_logging_obj.update_environment_variables(
|
||||
model=None,
|
||||
optional_params={"eval_id": eval_id, **query_params},
|
||||
litellm_params={
|
||||
"litellm_call_id": litellm_call_id,
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
# Make HTTP request
|
||||
response = base_llm_http_handler.list_runs_handler( # type: ignore
|
||||
url=url,
|
||||
query_params=query_params,
|
||||
evals_api_provider_config=evals_api_provider_config,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
extra_headers=headers,
|
||||
timeout=timeout or request_timeout,
|
||||
_is_async=_is_async,
|
||||
client=kwargs.get("client"),
|
||||
shared_session=kwargs.get("shared_session"),
|
||||
)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
async def aget_run(
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Run:
|
||||
"""
|
||||
Async: Get a specific run
|
||||
|
||||
Args:
|
||||
eval_id: The ID of the evaluation
|
||||
run_id: The ID of the run to retrieve
|
||||
extra_headers: Additional headers for the request
|
||||
extra_query: Additional query parameters
|
||||
timeout: Request timeout
|
||||
custom_llm_provider: Provider name (e.g., 'openai')
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
Run object
|
||||
"""
|
||||
local_vars = locals()
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
kwargs["aget_run"] = True
|
||||
|
||||
func = partial(
|
||||
get_run,
|
||||
eval_id=eval_id,
|
||||
run_id=run_id,
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
ctx = contextvars.copy_context()
|
||||
func_with_context = partial(ctx.run, func)
|
||||
init_response = await loop.run_in_executor(None, func_with_context)
|
||||
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
else:
|
||||
response = init_response
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
def get_run(
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Union[Run, Coroutine[Any, Any, Run]]:
|
||||
"""
|
||||
Get a specific run
|
||||
|
||||
Args:
|
||||
eval_id: The ID of the evaluation
|
||||
run_id: The ID of the run to retrieve
|
||||
extra_headers: Additional headers for the request
|
||||
extra_query: Additional query parameters
|
||||
timeout: Request timeout
|
||||
custom_llm_provider: Provider name (e.g., 'openai')
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
Run object
|
||||
"""
|
||||
local_vars = locals()
|
||||
try:
|
||||
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
|
||||
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
|
||||
_is_async = kwargs.pop("aget_run", False) is True
|
||||
|
||||
# Get LiteLLM parameters
|
||||
litellm_params = GenericLiteLLMParams(**kwargs)
|
||||
|
||||
# Determine provider
|
||||
if custom_llm_provider is None:
|
||||
custom_llm_provider = "openai"
|
||||
|
||||
# Get provider config
|
||||
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
|
||||
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
|
||||
provider=litellm.LlmProviders(custom_llm_provider),
|
||||
)
|
||||
)
|
||||
|
||||
if evals_api_provider_config is None:
|
||||
raise ValueError(f"GET run is not supported for {custom_llm_provider}")
|
||||
|
||||
# Validate environment and get headers
|
||||
headers = extra_headers or {}
|
||||
headers = evals_api_provider_config.validate_environment(
|
||||
headers=headers, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
# Transform request
|
||||
api_base = litellm_params.api_base or DEFAULT_OPENAI_API_BASE
|
||||
url, headers = evals_api_provider_config.transform_get_run_request(
|
||||
eval_id=eval_id,
|
||||
run_id=run_id,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Pre-call logging
|
||||
litellm_logging_obj.update_environment_variables(
|
||||
model=None,
|
||||
optional_params={"eval_id": eval_id, "run_id": run_id},
|
||||
litellm_params={
|
||||
"litellm_call_id": litellm_call_id,
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
# Make HTTP request
|
||||
response = base_llm_http_handler.get_run_handler( # type: ignore
|
||||
url=url,
|
||||
evals_api_provider_config=evals_api_provider_config,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
extra_headers=headers,
|
||||
timeout=timeout or request_timeout,
|
||||
_is_async=_is_async,
|
||||
client=kwargs.get("client"),
|
||||
shared_session=kwargs.get("shared_session"),
|
||||
)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
async def acancel_run(
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> CancelRunResponse:
|
||||
"""
|
||||
Async: Cancel a running run
|
||||
|
||||
Args:
|
||||
eval_id: The ID of the evaluation
|
||||
run_id: The ID of the run to cancel
|
||||
extra_headers: Additional headers for the request
|
||||
extra_query: Additional query parameters
|
||||
timeout: Request timeout
|
||||
custom_llm_provider: Provider name (e.g., 'openai')
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
CancelRunResponse object
|
||||
"""
|
||||
local_vars = locals()
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
kwargs["acancel_run"] = True
|
||||
|
||||
func = partial(
|
||||
cancel_run,
|
||||
eval_id=eval_id,
|
||||
run_id=run_id,
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
ctx = contextvars.copy_context()
|
||||
func_with_context = partial(ctx.run, func)
|
||||
init_response = await loop.run_in_executor(None, func_with_context)
|
||||
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
else:
|
||||
response = init_response
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
def cancel_run(
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Union[CancelRunResponse, Coroutine[Any, Any, CancelRunResponse]]:
|
||||
"""
|
||||
Cancel a running run
|
||||
|
||||
Args:
|
||||
eval_id: The ID of the evaluation
|
||||
run_id: The ID of the run to cancel
|
||||
extra_headers: Additional headers for the request
|
||||
extra_query: Additional query parameters
|
||||
timeout: Request timeout
|
||||
custom_llm_provider: Provider name (e.g., 'openai')
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
CancelRunResponse object
|
||||
"""
|
||||
local_vars = locals()
|
||||
try:
|
||||
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
|
||||
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
|
||||
_is_async = kwargs.pop("acancel_run", False) is True
|
||||
|
||||
# Get LiteLLM parameters
|
||||
litellm_params = GenericLiteLLMParams(**kwargs)
|
||||
|
||||
# Determine provider
|
||||
if custom_llm_provider is None:
|
||||
custom_llm_provider = "openai"
|
||||
|
||||
# Get provider config
|
||||
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
|
||||
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
|
||||
provider=litellm.LlmProviders(custom_llm_provider),
|
||||
)
|
||||
)
|
||||
|
||||
if evals_api_provider_config is None:
|
||||
raise ValueError(f"CANCEL run is not supported for {custom_llm_provider}")
|
||||
|
||||
# Validate environment and get headers
|
||||
headers = extra_headers or {}
|
||||
headers = evals_api_provider_config.validate_environment(
|
||||
headers=headers, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
# Transform request
|
||||
api_base = litellm_params.api_base or DEFAULT_OPENAI_API_BASE
|
||||
url, headers, request_body = evals_api_provider_config.transform_cancel_run_request(
|
||||
eval_id=eval_id,
|
||||
run_id=run_id,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Pre-call logging
|
||||
litellm_logging_obj.update_environment_variables(
|
||||
model=None,
|
||||
optional_params={"eval_id": eval_id, "run_id": run_id},
|
||||
litellm_params={
|
||||
"litellm_call_id": litellm_call_id,
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
# Make HTTP request
|
||||
response = base_llm_http_handler.cancel_run_handler( # type: ignore
|
||||
url=url,
|
||||
evals_api_provider_config=evals_api_provider_config,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
extra_headers=headers,
|
||||
timeout=timeout or request_timeout,
|
||||
_is_async=_is_async,
|
||||
client=kwargs.get("client"),
|
||||
shared_session=kwargs.get("shared_session"),
|
||||
)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
# ===================================
|
||||
# Delete Run API Functions
|
||||
# ===================================
|
||||
|
||||
|
||||
@client
|
||||
async def adelete_run(
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> RunDeleteResponse:
|
||||
"""
|
||||
Async: Delete a run
|
||||
|
||||
Args:
|
||||
eval_id: The ID of the evaluation
|
||||
run_id: The ID of the run to delete
|
||||
extra_headers: Additional headers for the request
|
||||
extra_query: Additional query parameters
|
||||
timeout: Request timeout
|
||||
custom_llm_provider: Provider name (e.g., 'openai')
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
RunDeleteResponse object
|
||||
"""
|
||||
local_vars = locals()
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
kwargs["adelete_run"] = True
|
||||
|
||||
func = partial(
|
||||
delete_run,
|
||||
eval_id=eval_id,
|
||||
run_id=run_id,
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
ctx = contextvars.copy_context()
|
||||
func_with_context = partial(ctx.run, func)
|
||||
init_response = await loop.run_in_executor(None, func_with_context)
|
||||
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
else:
|
||||
response = init_response
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
def delete_run(
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Union[RunDeleteResponse, Coroutine[Any, Any, RunDeleteResponse]]:
|
||||
"""
|
||||
Delete a run
|
||||
|
||||
Args:
|
||||
eval_id: The ID of the evaluation
|
||||
run_id: The ID of the run to delete
|
||||
extra_headers: Additional headers for the request
|
||||
extra_query: Additional query parameters
|
||||
timeout: Request timeout
|
||||
custom_llm_provider: Provider name (e.g., 'openai')
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
RunDeleteResponse object
|
||||
"""
|
||||
local_vars = locals()
|
||||
try:
|
||||
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
|
||||
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
|
||||
_is_async = kwargs.pop("adelete_run", False) is True
|
||||
|
||||
# Get LiteLLM parameters
|
||||
litellm_params = GenericLiteLLMParams(**kwargs)
|
||||
|
||||
# Determine provider
|
||||
if custom_llm_provider is None:
|
||||
custom_llm_provider = "openai"
|
||||
|
||||
# Get provider config
|
||||
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
|
||||
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
|
||||
provider=litellm.LlmProviders(custom_llm_provider),
|
||||
)
|
||||
)
|
||||
|
||||
if evals_api_provider_config is None:
|
||||
raise ValueError(f"DELETE run is not supported for {custom_llm_provider}")
|
||||
|
||||
# Validate environment and get headers
|
||||
headers = extra_headers or {}
|
||||
headers = evals_api_provider_config.validate_environment(
|
||||
headers=headers, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
# Transform request
|
||||
api_base = litellm_params.api_base or DEFAULT_OPENAI_API_BASE
|
||||
url, headers, request_body = evals_api_provider_config.transform_delete_run_request(
|
||||
eval_id=eval_id,
|
||||
run_id=run_id,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Pre-call logging
|
||||
litellm_logging_obj.update_environment_variables(
|
||||
model=None,
|
||||
optional_params={"eval_id": eval_id, "run_id": run_id},
|
||||
litellm_params={
|
||||
"litellm_call_id": litellm_call_id,
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
# Make HTTP request
|
||||
response = base_llm_http_handler.delete_run_handler( # type: ignore
|
||||
url=url,
|
||||
evals_api_provider_config=evals_api_provider_config,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
extra_headers=headers,
|
||||
timeout=timeout or request_timeout,
|
||||
_is_async=_is_async,
|
||||
client=kwargs.get("client"),
|
||||
shared_session=kwargs.get("shared_session"),
|
||||
)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
original_exception=e,
|
||||
completion_kwargs=local_vars,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -136,9 +136,13 @@ if TYPE_CHECKING:
|
|||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
from litellm.types.llms.openai_evals import (
|
||||
CancelEvalResponse,
|
||||
CancelRunResponse,
|
||||
DeleteEvalResponse,
|
||||
Eval,
|
||||
ListEvalsResponse,
|
||||
ListRunsResponse,
|
||||
Run,
|
||||
RunDeleteResponse,
|
||||
)
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
|
|
@ -9983,3 +9987,550 @@ class BaseLLMHTTPHandler:
|
|||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
# ===================================
|
||||
# Eval Runs API Handlers
|
||||
# ===================================
|
||||
|
||||
def create_run_handler(
|
||||
self,
|
||||
url: str,
|
||||
request_body: Dict,
|
||||
evals_api_provider_config: "BaseEvalsAPIConfig",
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
_is_async: bool = False,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> Union["Run", Coroutine[Any, Any, "Run"]]:
|
||||
"""Create a run"""
|
||||
if _is_async:
|
||||
return self.async_create_run_handler(
|
||||
url=url,
|
||||
request_body=request_body,
|
||||
evals_api_provider_config=evals_api_provider_config,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
|
||||
if client is None or not isinstance(client, HTTPHandler):
|
||||
sync_httpx_client = _get_httpx_client(
|
||||
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
|
||||
)
|
||||
else:
|
||||
sync_httpx_client = client
|
||||
|
||||
headers = extra_headers or {}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input=request_body.get("name", ""),
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": request_body,
|
||||
"api_base": url,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = sync_httpx_client.post(
|
||||
url=url, headers=headers, json=request_body, timeout=timeout
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
provider_config=evals_api_provider_config,
|
||||
)
|
||||
|
||||
return evals_api_provider_config.transform_create_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
async def async_create_run_handler(
|
||||
self,
|
||||
url: str,
|
||||
request_body: Dict,
|
||||
evals_api_provider_config: "BaseEvalsAPIConfig",
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> "Run":
|
||||
"""Async create a run"""
|
||||
if client is None or not isinstance(client, AsyncHTTPHandler):
|
||||
async_httpx_client = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders(custom_llm_provider),
|
||||
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
|
||||
)
|
||||
else:
|
||||
async_httpx_client = client
|
||||
|
||||
headers = extra_headers or {}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input=request_body.get("name", ""),
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": request_body,
|
||||
"api_base": url,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = await async_httpx_client.post(
|
||||
url=url, headers=headers, json=request_body, timeout=timeout
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
provider_config=evals_api_provider_config,
|
||||
)
|
||||
|
||||
return evals_api_provider_config.transform_create_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
def list_runs_handler(
|
||||
self,
|
||||
url: str,
|
||||
query_params: Dict,
|
||||
evals_api_provider_config: "BaseEvalsAPIConfig",
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
_is_async: bool = False,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> Union["ListRunsResponse", Coroutine[Any, Any, "ListRunsResponse"]]:
|
||||
"""List runs"""
|
||||
if _is_async:
|
||||
return self.async_list_runs_handler(
|
||||
url=url,
|
||||
query_params=query_params,
|
||||
evals_api_provider_config=evals_api_provider_config,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
|
||||
if client is None or not isinstance(client, HTTPHandler):
|
||||
sync_httpx_client = _get_httpx_client(
|
||||
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
|
||||
)
|
||||
else:
|
||||
sync_httpx_client = client
|
||||
|
||||
headers = extra_headers or {}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input="",
|
||||
api_key="",
|
||||
additional_args={
|
||||
"api_base": url,
|
||||
"headers": headers,
|
||||
"params": query_params,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = sync_httpx_client.get(
|
||||
url=url, headers=headers, params=query_params
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
provider_config=evals_api_provider_config,
|
||||
)
|
||||
|
||||
return evals_api_provider_config.transform_list_runs_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
async def async_list_runs_handler(
|
||||
self,
|
||||
url: str,
|
||||
query_params: Dict,
|
||||
evals_api_provider_config: "BaseEvalsAPIConfig",
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> "ListRunsResponse":
|
||||
"""Async list runs"""
|
||||
if client is None or not isinstance(client, AsyncHTTPHandler):
|
||||
async_httpx_client = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders(custom_llm_provider),
|
||||
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
|
||||
)
|
||||
else:
|
||||
async_httpx_client = client
|
||||
|
||||
headers = extra_headers or {}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input="",
|
||||
api_key="",
|
||||
additional_args={
|
||||
"api_base": url,
|
||||
"headers": headers,
|
||||
"params": query_params,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = await async_httpx_client.get(
|
||||
url=url, headers=headers, params=query_params
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
provider_config=evals_api_provider_config,
|
||||
)
|
||||
|
||||
return evals_api_provider_config.transform_list_runs_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
def get_run_handler(
|
||||
self,
|
||||
url: str,
|
||||
evals_api_provider_config: "BaseEvalsAPIConfig",
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
_is_async: bool = False,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> Union["Run", Coroutine[Any, Any, "Run"]]:
|
||||
"""Get a run"""
|
||||
if _is_async:
|
||||
return self.async_get_run_handler(
|
||||
url=url,
|
||||
evals_api_provider_config=evals_api_provider_config,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
|
||||
if client is None or not isinstance(client, HTTPHandler):
|
||||
sync_httpx_client = _get_httpx_client(
|
||||
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
|
||||
)
|
||||
else:
|
||||
sync_httpx_client = client
|
||||
|
||||
headers = extra_headers or {}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input="",
|
||||
api_key="",
|
||||
additional_args={
|
||||
"api_base": url,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = sync_httpx_client.get(url=url, headers=headers)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
provider_config=evals_api_provider_config,
|
||||
)
|
||||
|
||||
return evals_api_provider_config.transform_get_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
async def async_get_run_handler(
|
||||
self,
|
||||
url: str,
|
||||
evals_api_provider_config: "BaseEvalsAPIConfig",
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> "Run":
|
||||
"""Async get a run"""
|
||||
if client is None or not isinstance(client, AsyncHTTPHandler):
|
||||
async_httpx_client = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders(custom_llm_provider),
|
||||
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
|
||||
)
|
||||
else:
|
||||
async_httpx_client = client
|
||||
|
||||
headers = extra_headers or {}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input="",
|
||||
api_key="",
|
||||
additional_args={
|
||||
"api_base": url,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = await async_httpx_client.get(
|
||||
url=url, headers=headers
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
provider_config=evals_api_provider_config,
|
||||
)
|
||||
|
||||
return evals_api_provider_config.transform_get_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
def cancel_run_handler(
|
||||
self,
|
||||
url: str,
|
||||
evals_api_provider_config: "BaseEvalsAPIConfig",
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
_is_async: bool = False,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> Union["CancelRunResponse", Coroutine[Any, Any, "CancelRunResponse"]]:
|
||||
"""Cancel a run"""
|
||||
if _is_async:
|
||||
return self.async_cancel_run_handler(
|
||||
url=url,
|
||||
evals_api_provider_config=evals_api_provider_config,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
|
||||
if client is None or not isinstance(client, HTTPHandler):
|
||||
sync_httpx_client = _get_httpx_client(
|
||||
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
|
||||
)
|
||||
else:
|
||||
sync_httpx_client = client
|
||||
|
||||
headers = extra_headers or {}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input="",
|
||||
api_key="",
|
||||
additional_args={
|
||||
"api_base": url,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = sync_httpx_client.post(
|
||||
url=url, headers=headers, json={}, timeout=timeout
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
provider_config=evals_api_provider_config,
|
||||
)
|
||||
|
||||
return evals_api_provider_config.transform_cancel_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
async def async_cancel_run_handler(
|
||||
self,
|
||||
url: str,
|
||||
evals_api_provider_config: "BaseEvalsAPIConfig",
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> "CancelRunResponse":
|
||||
"""Async cancel a run"""
|
||||
if client is None or not isinstance(client, AsyncHTTPHandler):
|
||||
async_httpx_client = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders(custom_llm_provider),
|
||||
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
|
||||
)
|
||||
else:
|
||||
async_httpx_client = client
|
||||
|
||||
headers = extra_headers or {}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input="",
|
||||
api_key="",
|
||||
additional_args={
|
||||
"api_base": url,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = await async_httpx_client.post(
|
||||
url=url, headers=headers, json={}, timeout=timeout
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
provider_config=evals_api_provider_config,
|
||||
)
|
||||
|
||||
return evals_api_provider_config.transform_cancel_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
def delete_run_handler(
|
||||
self,
|
||||
url: str,
|
||||
evals_api_provider_config: "BaseEvalsAPIConfig",
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
_is_async: bool = False,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> Union["RunDeleteResponse", Coroutine[Any, Any, "RunDeleteResponse"]]:
|
||||
"""Delete a run"""
|
||||
if _is_async:
|
||||
return self.async_delete_run_handler(
|
||||
url=url,
|
||||
evals_api_provider_config=evals_api_provider_config,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
|
||||
if client is None or not isinstance(client, HTTPHandler):
|
||||
sync_httpx_client = _get_httpx_client(
|
||||
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
|
||||
)
|
||||
else:
|
||||
sync_httpx_client = client
|
||||
|
||||
headers = extra_headers or {}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input="",
|
||||
api_key="",
|
||||
additional_args={
|
||||
"api_base": url,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = sync_httpx_client.delete(
|
||||
url=url, headers=headers, timeout=timeout
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
provider_config=evals_api_provider_config,
|
||||
)
|
||||
|
||||
return evals_api_provider_config.transform_delete_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
async def async_delete_run_handler(
|
||||
self,
|
||||
url: str,
|
||||
evals_api_provider_config: "BaseEvalsAPIConfig",
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
extra_headers: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> "RunDeleteResponse":
|
||||
"""Async delete a run"""
|
||||
if client is None or not isinstance(client, AsyncHTTPHandler):
|
||||
async_httpx_client = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders(custom_llm_provider),
|
||||
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
|
||||
)
|
||||
else:
|
||||
async_httpx_client = client
|
||||
|
||||
headers = extra_headers or {}
|
||||
|
||||
logging_obj.pre_call(
|
||||
input="",
|
||||
api_key="",
|
||||
additional_args={
|
||||
"api_base": url,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = await async_httpx_client.delete(
|
||||
url=url, headers=headers, timeout=timeout
|
||||
)
|
||||
except Exception as e:
|
||||
raise self._handle_error(
|
||||
e=e,
|
||||
provider_config=evals_api_provider_config,
|
||||
)
|
||||
|
||||
return evals_api_provider_config.transform_delete_run_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -720,6 +720,11 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
"acreate_run",
|
||||
"alist_runs",
|
||||
"aget_run",
|
||||
"acancel_run",
|
||||
"adelete_run",
|
||||
],
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
general_settings: dict,
|
||||
|
|
|
|||
|
|
@ -80,6 +80,12 @@ ROUTE_ENDPOINT_MAPPING = {
|
|||
"aupdate_eval": "/evals/{eval_id}",
|
||||
"adelete_eval": "/evals/{eval_id}",
|
||||
"acancel_eval": "/evals/{eval_id}/cancel",
|
||||
# OpenAI Evals Runs API routes
|
||||
"acreate_run": "/evals/{eval_id}/runs",
|
||||
"alist_runs": "/evals/{eval_id}/runs",
|
||||
"aget_run": "/evals/{eval_id}/runs/{run_id}",
|
||||
"acancel_run": "/evals/{eval_id}/runs/{run_id}/cancel",
|
||||
"adelete_run": "/evals/{eval_id}/runs/{run_id}",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -203,6 +209,11 @@ async def route_request(
|
|||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
"acreate_run",
|
||||
"alist_runs",
|
||||
"aget_run",
|
||||
"acancel_run",
|
||||
"adelete_run",
|
||||
],
|
||||
):
|
||||
"""
|
||||
|
|
@ -278,6 +289,11 @@ async def route_request(
|
|||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
"acreate_run",
|
||||
"alist_runs",
|
||||
"aget_run",
|
||||
"acancel_run",
|
||||
"adelete_run",
|
||||
]:
|
||||
# If a model is provided, get its credentials from the router
|
||||
model = data.get("model")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue