mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
[Feat] Add vllm batch+files API support (#15823)
* add OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS * fix use OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS * add _get_batch_job_total_usage_from_file_content * fixes for vLLM + 12 labs async invoke * fix: vLLM Batch APIs * afile_retrieve * test_hosted_vllm_full_workflow * fix SERVER_URL for test
This commit is contained in:
Ishaan Jaff
GitHub
parent
9ff2ecc16d
commit
1bb9e1bde8
7 changed files with 591 additions and 63 deletions
|
|
@ -14,8 +14,7 @@ from litellm.utils import token_counter
|
|||
|
||||
async def calculate_batch_cost_and_usage(
|
||||
file_content_dictionary: List[dict],
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"],
|
||||
model_name: Optional[str] = None,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"],
|
||||
) -> Tuple[float, Usage, List[str]]:
|
||||
"""
|
||||
Calculate the cost and usage of a batch
|
||||
|
|
@ -37,8 +36,7 @@ async def calculate_batch_cost_and_usage(
|
|||
|
||||
async def _handle_completed_batch(
|
||||
batch: Batch,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"],
|
||||
model_name: Optional[str] = None,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"],
|
||||
) -> Tuple[float, Usage, List[str]]:
|
||||
"""Helper function to process a completed batch and handle logging"""
|
||||
# Get batch results
|
||||
|
|
@ -84,8 +82,7 @@ def _get_batch_models_from_file_content(
|
|||
|
||||
def _batch_cost_calculator(
|
||||
file_content_dictionary: List[dict],
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
|
||||
model_name: Optional[str] = None,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
|
||||
) -> float:
|
||||
"""
|
||||
Calculate the cost of a batch based on the output file id
|
||||
|
|
@ -186,7 +183,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
|
|||
|
||||
async def _get_batch_output_file_content_as_dictionary(
|
||||
batch: Batch,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
|
||||
) -> List[dict]:
|
||||
"""
|
||||
Get the batch output file content as a list of dictionaries
|
||||
|
|
@ -225,7 +222,7 @@ def _get_file_content_as_dictionary(file_content: bytes) -> List[dict]:
|
|||
|
||||
def _get_batch_job_cost_from_file_content(
|
||||
file_content_dictionary: List[dict],
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
|
||||
) -> float:
|
||||
"""
|
||||
Get the cost of a batch job from the file content
|
||||
|
|
@ -253,8 +250,7 @@ def _get_batch_job_cost_from_file_content(
|
|||
|
||||
def _get_batch_job_total_usage_from_file_content(
|
||||
file_content_dictionary: List[dict],
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
|
||||
model_name: Optional[str] = None,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
|
||||
) -> Usage:
|
||||
"""
|
||||
Get the tokens of a batch job from the file content
|
||||
|
|
|
|||
|
|
@ -18,11 +18,14 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
|
|||
|
||||
import httpx
|
||||
from openai.types.batch import BatchRequestCounts
|
||||
from openai.types.batch import Metadata
|
||||
from openai.types.batch import Metadata as OpenAIBatchMetadata
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.azure.batches.handler import AzureBatchesAPI
|
||||
from litellm.llms.bedrock.batches.handler import BedrockBatchesHandler
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.openai.openai import OpenAIBatchesAPI
|
||||
|
|
@ -35,7 +38,11 @@ from litellm.types.llms.openai import (
|
|||
RetrieveBatchRequest,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LiteLLMBatch, LlmProviders
|
||||
from litellm.types.utils import (
|
||||
OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS,
|
||||
LiteLLMBatch,
|
||||
LlmProviders,
|
||||
)
|
||||
from litellm.utils import (
|
||||
ProviderConfigManager,
|
||||
client,
|
||||
|
|
@ -100,7 +107,7 @@ async def acreate_batch(
|
|||
completion_window: Literal["24h"],
|
||||
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
|
||||
input_file_id: str,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
|
||||
metadata: Optional[Dict[str, str]] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
|
|
@ -148,7 +155,7 @@ def create_batch(
|
|||
completion_window: Literal["24h"],
|
||||
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
|
||||
input_file_id: str,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
|
||||
metadata: Optional[Dict[str, str]] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
|
|
@ -235,7 +242,7 @@ def create_batch(
|
|||
)
|
||||
return response
|
||||
api_base: Optional[str] = None
|
||||
if custom_llm_provider == "openai":
|
||||
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
|
||||
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
api_base = (
|
||||
optional_params.api_base
|
||||
|
|
@ -350,7 +357,7 @@ def create_batch(
|
|||
@client
|
||||
async def aretrieve_batch(
|
||||
batch_id: str,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
|
||||
metadata: Optional[Dict[str, str]] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
|
|
@ -396,10 +403,10 @@ def _handle_retrieve_batch_providers_without_provider_config(
|
|||
litellm_params: dict,
|
||||
_retrieve_batch_request: RetrieveBatchRequest,
|
||||
_is_async: bool,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
|
||||
):
|
||||
api_base: Optional[str] = None
|
||||
if custom_llm_provider == "openai":
|
||||
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
|
||||
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
api_base = (
|
||||
optional_params.api_base
|
||||
|
|
@ -512,7 +519,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
|
|||
@client
|
||||
def retrieve_batch(
|
||||
batch_id: str,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
|
||||
metadata: Optional[Dict[str, str]] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
|
|
@ -576,7 +583,7 @@ def retrieve_batch(
|
|||
async_kwargs = kwargs.copy()
|
||||
async_kwargs.pop("aws_region_name", None)
|
||||
|
||||
return _handle_async_invoke_status(
|
||||
return BedrockBatchesHandler._handle_async_invoke_status(
|
||||
batch_id=batch_id,
|
||||
aws_region_name=kwargs.get("aws_region_name", "us-east-1"),
|
||||
logging_obj=litellm_logging_obj,
|
||||
|
|
@ -644,7 +651,7 @@ def retrieve_batch(
|
|||
async def alist_batches(
|
||||
after: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "vertex_ai"] = "openai",
|
||||
metadata: Optional[Dict[str, str]] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
|
|
@ -687,7 +694,7 @@ async def alist_batches(
|
|||
def list_batches(
|
||||
after: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "vertex_ai"] = "openai",
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
**kwargs,
|
||||
|
|
@ -727,7 +734,7 @@ def list_batches(
|
|||
timeout = 600.0
|
||||
|
||||
_is_async = kwargs.pop("alist_batches", False) is True
|
||||
if custom_llm_provider == "openai":
|
||||
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
|
||||
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
api_base = (
|
||||
optional_params.api_base
|
||||
|
|
@ -928,7 +935,7 @@ def cancel_batch(
|
|||
|
||||
_is_async = kwargs.pop("acancel_batch", False) is True
|
||||
api_base: Optional[str] = None
|
||||
if custom_llm_provider == "openai":
|
||||
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
|
||||
api_base = (
|
||||
optional_params.api_base
|
||||
or litellm.api_base
|
||||
|
|
@ -1043,19 +1050,20 @@ def _handle_async_invoke_status(
|
|||
)
|
||||
|
||||
# Transform response to a LiteLLMBatch object
|
||||
from litellm.types.llms.openai import BatchJobStatus
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
# Normalize status to lowercase (AWS returns 'Completed', 'Failed', etc.)
|
||||
aws_status_raw = status_response.get("status", "")
|
||||
aws_status_lower = aws_status_raw.lower()
|
||||
# Map AWS status values to LiteLLM expected values
|
||||
status_mapping = {
|
||||
status_mapping: dict[str, BatchJobStatus] = {
|
||||
"completed": "completed",
|
||||
"failed": "failed",
|
||||
"inprogress": "in_progress",
|
||||
"in_progress": "in_progress",
|
||||
}
|
||||
normalized_status = status_mapping.get(aws_status_lower, aws_status_lower)
|
||||
normalized_status: BatchJobStatus = status_mapping.get(aws_status_lower, "failed") # Default to "failed" if unknown status
|
||||
|
||||
# Get output S3 URI safely
|
||||
output_s3_uri = ""
|
||||
|
|
@ -1065,13 +1073,15 @@ def _handle_async_invoke_status(
|
|||
pass
|
||||
|
||||
# Use BedrockBatchesConfig's timestamp parsing method (expects raw AWS status string)
|
||||
import time
|
||||
|
||||
from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig
|
||||
created_at, in_progress_at, completed_at, failed_at, _, _ = BedrockBatchesConfig()._parse_timestamps_and_status(status_response, aws_status_raw)
|
||||
result = LiteLLMBatch(
|
||||
id=status_response["invocationArn"],
|
||||
object="batch",
|
||||
status=normalized_status,
|
||||
created_at=created_at,
|
||||
created_at=created_at or int(time.time()), # Provide default timestamp if None
|
||||
in_progress_at=in_progress_at,
|
||||
completed_at=completed_at,
|
||||
failed_at=failed_at,
|
||||
|
|
|
|||
|
|
@ -30,7 +30,10 @@ from litellm.types.llms.openai import (
|
|||
OpenAIFileObject,
|
||||
)
|
||||
from litellm.types.router import *
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.types.utils import (
|
||||
OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS,
|
||||
LlmProviders,
|
||||
)
|
||||
from litellm.utils import (
|
||||
ProviderConfigManager,
|
||||
client,
|
||||
|
|
@ -51,7 +54,7 @@ vertex_ai_files_instance = VertexAIFilesHandler()
|
|||
async def acreate_file(
|
||||
file: FileTypes,
|
||||
purpose: Literal["assistants", "batch", "fine-tune"],
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
**kwargs,
|
||||
|
|
@ -95,9 +98,7 @@ async def acreate_file(
|
|||
def create_file(
|
||||
file: FileTypes,
|
||||
purpose: Literal["assistants", "batch", "fine-tune"],
|
||||
custom_llm_provider: Optional[
|
||||
Literal["openai", "azure", "vertex_ai", "bedrock"]
|
||||
] = None,
|
||||
custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"]] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
**kwargs,
|
||||
|
|
@ -165,7 +166,7 @@ def create_file(
|
|||
),
|
||||
timeout=timeout,
|
||||
)
|
||||
elif custom_llm_provider == "openai":
|
||||
elif custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
|
||||
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
api_base = (
|
||||
optional_params.api_base
|
||||
|
|
@ -276,7 +277,7 @@ def create_file(
|
|||
@client
|
||||
async def afile_retrieve(
|
||||
file_id: str,
|
||||
custom_llm_provider: Literal["openai", "azure"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai",
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
**kwargs,
|
||||
|
|
@ -317,7 +318,7 @@ async def afile_retrieve(
|
|||
@client
|
||||
def file_retrieve(
|
||||
file_id: str,
|
||||
custom_llm_provider: Literal["openai", "azure"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai",
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
**kwargs,
|
||||
|
|
@ -347,7 +348,7 @@ def file_retrieve(
|
|||
|
||||
_is_async = kwargs.pop("is_async", False) is True
|
||||
|
||||
if custom_llm_provider == "openai":
|
||||
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
|
||||
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
api_base = (
|
||||
optional_params.api_base
|
||||
|
|
@ -514,7 +515,7 @@ def file_delete(
|
|||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
_is_async = kwargs.pop("is_async", False) is True
|
||||
if custom_llm_provider == "openai":
|
||||
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
|
||||
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
api_base = (
|
||||
optional_params.api_base
|
||||
|
|
@ -670,7 +671,7 @@ def file_list(
|
|||
timeout = 600.0
|
||||
|
||||
_is_async = kwargs.pop("is_async", False) is True
|
||||
if custom_llm_provider == "openai":
|
||||
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
|
||||
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
api_base = (
|
||||
optional_params.api_base
|
||||
|
|
@ -754,7 +755,7 @@ def file_list(
|
|||
@client
|
||||
async def afile_content(
|
||||
file_id: str,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
**kwargs,
|
||||
|
|
@ -799,7 +800,7 @@ def file_content(
|
|||
file_id: str,
|
||||
model: Optional[str] = None,
|
||||
custom_llm_provider: Optional[
|
||||
Union[Literal["openai", "azure", "vertex_ai"], str]
|
||||
Union[Literal["openai", "azure", "vertex_ai", "hosted_vllm"], str]
|
||||
] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
|
|
@ -846,7 +847,7 @@ def file_content(
|
|||
|
||||
_is_async = kwargs.pop("afile_content", False) is True
|
||||
|
||||
if custom_llm_provider == "openai":
|
||||
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
|
||||
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
api_base = (
|
||||
optional_params.api_base
|
||||
|
|
|
|||
96
litellm/llms/bedrock/batches/handler.py
Normal file
96
litellm/llms/bedrock/batches/handler.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
from openai.types.batch import BatchRequestCounts
|
||||
from openai.types.batch import Metadata as OpenAIBatchMetadata
|
||||
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
|
||||
class BedrockBatchesHandler:
|
||||
"""
|
||||
Handler for Bedrock Batches.
|
||||
|
||||
Specific providers/models needed some special handling.
|
||||
|
||||
E.g. Twelve Labs Embedding Async Invoke
|
||||
"""
|
||||
@staticmethod
|
||||
def _handle_async_invoke_status(
|
||||
batch_id: str, aws_region_name: str, logging_obj=None, **kwargs
|
||||
) -> "LiteLLMBatch":
|
||||
"""
|
||||
Handle async invoke status check for AWS Bedrock.
|
||||
|
||||
This is for Twelve Labs Embedding Async Invoke.
|
||||
|
||||
Args:
|
||||
batch_id: The async invoke ARN
|
||||
aws_region_name: AWS region name
|
||||
**kwargs: Additional parameters
|
||||
|
||||
Returns:
|
||||
dict: Status information including status, output_file_id (S3 URL), etc.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
from litellm.llms.bedrock.embed.embedding import BedrockEmbedding
|
||||
|
||||
async def _async_get_status():
|
||||
# Create embedding handler instance
|
||||
embedding_handler = BedrockEmbedding()
|
||||
|
||||
# Get the status of the async invoke job
|
||||
status_response = await embedding_handler._get_async_invoke_status(
|
||||
invocation_arn=batch_id,
|
||||
aws_region_name=aws_region_name,
|
||||
logging_obj=logging_obj,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Transform response to a LiteLLMBatch object
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
openai_batch_metadata: OpenAIBatchMetadata = {
|
||||
"output_file_id": status_response["outputDataConfig"][
|
||||
"s3OutputDataConfig"
|
||||
]["s3Uri"],
|
||||
"failure_message": status_response.get("failureMessage") or "",
|
||||
"model_arn": status_response["modelArn"],
|
||||
}
|
||||
|
||||
result = LiteLLMBatch(
|
||||
id=status_response["invocationArn"],
|
||||
object="batch",
|
||||
status=status_response["status"],
|
||||
created_at=status_response["submitTime"],
|
||||
in_progress_at=status_response["lastModifiedTime"],
|
||||
completed_at=status_response.get("endTime"),
|
||||
failed_at=status_response.get("endTime")
|
||||
if status_response["status"] == "failed"
|
||||
else None,
|
||||
request_counts=BatchRequestCounts(
|
||||
total=1,
|
||||
completed=1 if status_response["status"] == "completed" else 0,
|
||||
failed=1 if status_response["status"] == "failed" else 0,
|
||||
),
|
||||
metadata=openai_batch_metadata,
|
||||
completion_window="24h",
|
||||
endpoint="/v1/embeddings",
|
||||
input_file_id="",
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
# Since this function is called from within an async context via run_in_executor,
|
||||
# we need to create a new event loop in a thread to avoid conflicts
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
return new_loop.run_until_complete(_async_get_status())
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
return future.result()
|
||||
|
|
@ -6717,6 +6717,33 @@
|
|||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 159
|
||||
},
|
||||
"claude-opus-4-5": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"tool_use_system_prompt_tokens": 159
|
||||
},
|
||||
"claude-sonnet-4-20250514": {
|
||||
"deprecation_date": "2026-05-14",
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -7824,26 +7851,298 @@
|
|||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
|
||||
},
|
||||
"databricks/databricks-claude-3-7-sonnet": {
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"input_dbu_cost_per_token": 3.571e-05,
|
||||
"input_cost_per_token": 2.9999900000000002e-06,
|
||||
"input_dbu_cost_per_token": 4.2857e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 200000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Claude 3.7 conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.7857e-05,
|
||||
"output_db_cost_per_token": 0.000214286,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"output_cost_per_token": 1.5000020000000002e-05,
|
||||
"output_dbu_cost_per_token": 0.000214286,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-claude-haiku-4-5": {
|
||||
"input_cost_per_token": 1.00002e-06,
|
||||
"input_dbu_cost_per_token": 1.4286e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 200000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5.00003e-06,
|
||||
"output_dbu_cost_per_token": 7.1429e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-claude-opus-4": {
|
||||
"input_cost_per_token": 1.5000020000000002e-05,
|
||||
"input_dbu_cost_per_token": 0.000214286,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 32000,
|
||||
"max_tokens": 200000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.500003000000001e-05,
|
||||
"output_dbu_cost_per_token": 0.001071429,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-claude-opus-4-1": {
|
||||
"input_cost_per_token": 1.5000020000000002e-05,
|
||||
"input_dbu_cost_per_token": 0.000214286,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 32000,
|
||||
"max_tokens": 200000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.500003000000001e-05,
|
||||
"output_dbu_cost_per_token": 0.001071429,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-claude-opus-4-5": {
|
||||
"input_cost_per_token": 5.00003e-06,
|
||||
"input_dbu_cost_per_token": 7.1429e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 200000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5000010000000002e-05,
|
||||
"output_dbu_cost_per_token": 0.000357143,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-claude-sonnet-4": {
|
||||
"input_cost_per_token": 2.9999900000000002e-06,
|
||||
"input_dbu_cost_per_token": 4.2857e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 200000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5000020000000002e-05,
|
||||
"output_dbu_cost_per_token": 0.000214286,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-claude-sonnet-4-1": {
|
||||
"input_cost_per_token": 2.9999900000000002e-06,
|
||||
"input_dbu_cost_per_token": 4.2857e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 200000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5000020000000002e-05,
|
||||
"output_dbu_cost_per_token": 0.000214286,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-claude-sonnet-4-5": {
|
||||
"input_cost_per_token": 2.9999900000000002e-06,
|
||||
"input_dbu_cost_per_token": 4.2857e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"max_tokens": 200000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5000020000000002e-05,
|
||||
"output_dbu_cost_per_token": 0.000214286,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-gemini-2-5-flash": {
|
||||
"input_cost_per_token": 3.0001999999999996e-07,
|
||||
"input_dbu_cost_per_token": 4.285999999999999e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65535,
|
||||
"max_tokens": 1048576,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.49998e-06,
|
||||
"output_dbu_cost_per_token": 3.5714e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-gemini-2-5-pro": {
|
||||
"input_cost_per_token": 1.24999e-06,
|
||||
"input_dbu_cost_per_token": 1.7857e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 1048576,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9.999990000000002e-06,
|
||||
"output_dbu_cost_per_token": 0.000142857,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-gemma-3-12b": {
|
||||
"input_cost_per_token": 1.5000999999999998e-07,
|
||||
"input_dbu_cost_per_token": 2.1429999999999996e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 32000,
|
||||
"max_tokens": 128000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5.0001e-07,
|
||||
"output_dbu_cost_per_token": 7.143e-06,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
|
||||
},
|
||||
"databricks/databricks-gpt-5": {
|
||||
"input_cost_per_token": 1.24999e-06,
|
||||
"input_dbu_cost_per_token": 1.7857e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 400000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9.999990000000002e-06,
|
||||
"output_dbu_cost_per_token": 0.000142857,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
|
||||
},
|
||||
"databricks/databricks-gpt-5-1": {
|
||||
"input_cost_per_token": 1.24999e-06,
|
||||
"input_dbu_cost_per_token": 1.7857e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 400000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9.999990000000002e-06,
|
||||
"output_dbu_cost_per_token": 0.000142857,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
|
||||
},
|
||||
"databricks/databricks-gpt-5-mini": {
|
||||
"input_cost_per_token": 2.4997000000000006e-07,
|
||||
"input_dbu_cost_per_token": 3.571e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 400000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.9999700000000004e-06,
|
||||
"output_dbu_cost_per_token": 2.8571e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
|
||||
},
|
||||
"databricks/databricks-gpt-5-nano": {
|
||||
"input_cost_per_token": 4.998e-08,
|
||||
"input_dbu_cost_per_token": 7.14e-07,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 400000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.9998000000000007e-07,
|
||||
"output_dbu_cost_per_token": 5.714000000000001e-06,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
|
||||
},
|
||||
"databricks/databricks-gpt-oss-120b": {
|
||||
"input_cost_per_token": 1.5000999999999998e-07,
|
||||
"input_dbu_cost_per_token": 2.1429999999999996e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5.9997e-07,
|
||||
"output_dbu_cost_per_token": 8.571e-06,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
|
||||
},
|
||||
"databricks/databricks-gpt-oss-20b": {
|
||||
"input_cost_per_token": 7e-08,
|
||||
"input_dbu_cost_per_token": 1e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.0001999999999996e-07,
|
||||
"output_dbu_cost_per_token": 4.285999999999999e-06,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
|
||||
},
|
||||
"databricks/databricks-gte-large-en": {
|
||||
"input_cost_per_token": 1.2999e-07,
|
||||
"input_cost_per_token": 1.2999000000000001e-07,
|
||||
"input_dbu_cost_per_token": 1.857e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 8192,
|
||||
|
|
@ -7868,14 +8167,14 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-06,
|
||||
"output_cost_per_token": 1.5000300000000002e-06,
|
||||
"output_dbu_cost_per_token": 2.1429e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-llama-4-maverick": {
|
||||
"input_cost_per_token": 5e-06,
|
||||
"input_dbu_cost_per_token": 7.143e-05,
|
||||
"input_cost_per_token": 5.0001e-07,
|
||||
"input_dbu_cost_per_token": 7.143e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -7884,13 +8183,13 @@
|
|||
"notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_dbu_cost_per_token": 0.00021429,
|
||||
"output_cost_per_token": 1.5000300000000002e-06,
|
||||
"output_dbu_cost_per_token": 2.1429e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-meta-llama-3-1-405b-instruct": {
|
||||
"input_cost_per_token": 5e-06,
|
||||
"input_cost_per_token": 5.00003e-06,
|
||||
"input_dbu_cost_per_token": 7.1429e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -7900,14 +8199,29 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.500002e-05,
|
||||
"output_db_cost_per_token": 0.000214286,
|
||||
"output_cost_per_token": 1.5000020000000002e-05,
|
||||
"output_dbu_cost_per_token": 0.000214286,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-meta-llama-3-1-8b-instruct": {
|
||||
"input_cost_per_token": 1.5000999999999998e-07,
|
||||
"input_dbu_cost_per_token": 2.1429999999999996e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 200000,
|
||||
"metadata": {
|
||||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.5003000000000007e-07,
|
||||
"output_dbu_cost_per_token": 6.429000000000001e-06,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
|
||||
},
|
||||
"databricks/databricks-meta-llama-3-3-70b-instruct": {
|
||||
"input_cost_per_token": 1.00002e-06,
|
||||
"input_dbu_cost_per_token": 1.4286e-05,
|
||||
"input_cost_per_token": 5.0001e-07,
|
||||
"input_dbu_cost_per_token": 7.143e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -7916,8 +8230,8 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.99999e-06,
|
||||
"output_dbu_cost_per_token": 4.2857e-05,
|
||||
"output_cost_per_token": 1.5000300000000002e-06,
|
||||
"output_dbu_cost_per_token": 2.1429e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -7932,7 +8246,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.99999e-06,
|
||||
"output_cost_per_token": 2.9999900000000002e-06,
|
||||
"output_dbu_cost_per_token": 4.2857e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -7948,13 +8262,13 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9.9902e-07,
|
||||
"output_cost_per_token": 1.00002e-06,
|
||||
"output_dbu_cost_per_token": 1.4286e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"databricks/databricks-mpt-30b-instruct": {
|
||||
"input_cost_per_token": 9.9902e-07,
|
||||
"input_cost_per_token": 1.00002e-06,
|
||||
"input_dbu_cost_per_token": 1.4286e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 8192,
|
||||
|
|
@ -7964,7 +8278,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9.9902e-07,
|
||||
"output_cost_per_token": 1.00002e-06,
|
||||
"output_dbu_cost_per_token": 1.4286e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supports_tool_choice": true
|
||||
|
|
|
|||
|
|
@ -2660,6 +2660,12 @@ class LlmProviders(str, Enum):
|
|||
# Create a set of all provider values for quick lookup
|
||||
LlmProvidersSet = {provider.value for provider in LlmProviders}
|
||||
|
||||
# File and Batch API providers that are OpenAI-compatible
|
||||
OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: set[str] = {
|
||||
LlmProviders.OPENAI.value,
|
||||
LlmProviders.HOSTED_VLLM.value,
|
||||
}
|
||||
|
||||
|
||||
class SearchProviders(str, Enum):
|
||||
"""
|
||||
|
|
|
|||
105
tests/batches_tests/test_hosted_vllm_batches_and_files.py
Normal file
105
tests/batches_tests/test_hosted_vllm_batches_and_files.py
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
"""
|
||||
Unit Tests for hosted_vllm Batches and Files API
|
||||
|
||||
Tests the integration of hosted_vllm provider with LiteLLM's batch and file operations.
|
||||
Tests against a real OpenAI-compatible endpoint.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
)
|
||||
|
||||
import litellm
|
||||
|
||||
|
||||
SERVER_URL = "https://exampleopenaiendpoint-production-0ee2.up.railway.app/v1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
async def test_hosted_vllm_full_workflow():
|
||||
"""
|
||||
Test the complete workflow: create file -> create batch -> retrieve batch -> retrieve file.
|
||||
Tests against real OpenAI-compatible endpoint.
|
||||
"""
|
||||
litellm._turn_on_debug()
|
||||
file_name = "openai_batch_completions.jsonl"
|
||||
_current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
file_path = os.path.join(_current_dir, file_name)
|
||||
|
||||
# Step 1: Create file
|
||||
print("\n=== Step 1: Creating file ===")
|
||||
file_obj = await litellm.acreate_file(
|
||||
file=open(file_path, "rb"),
|
||||
purpose="batch",
|
||||
custom_llm_provider="hosted_vllm",
|
||||
api_base=SERVER_URL,
|
||||
api_key="test-api-key",
|
||||
)
|
||||
|
||||
print(f"✓ Created file: {file_obj.id}")
|
||||
assert file_obj.id is not None
|
||||
assert file_obj.object == "file"
|
||||
assert file_obj.purpose == "batch"
|
||||
|
||||
# Step 2: Create batch
|
||||
print("\n=== Step 2: Creating batch ===")
|
||||
batch_obj = await litellm.acreate_batch(
|
||||
completion_window="24h",
|
||||
endpoint="/v1/chat/completions",
|
||||
input_file_id=file_obj.id,
|
||||
custom_llm_provider="hosted_vllm",
|
||||
metadata={"test": "hosted_vllm_integration"},
|
||||
api_base=SERVER_URL,
|
||||
api_key="test-api-key",
|
||||
)
|
||||
|
||||
print(f"✓ Created batch: {batch_obj.id}")
|
||||
print(f" Status: {batch_obj.status}")
|
||||
print(f" Input file: {batch_obj.input_file_id}")
|
||||
assert batch_obj.id is not None
|
||||
assert batch_obj.object == "batch"
|
||||
assert batch_obj.input_file_id == file_obj.id
|
||||
assert batch_obj.endpoint == "/v1/chat/completions"
|
||||
|
||||
# Step 3: Retrieve batch
|
||||
print("\n=== Step 3: Retrieving batch ===")
|
||||
retrieved_batch = await litellm.aretrieve_batch(
|
||||
batch_id=batch_obj.id,
|
||||
custom_llm_provider="hosted_vllm",
|
||||
api_base=SERVER_URL,
|
||||
api_key="test-api-key",
|
||||
)
|
||||
|
||||
print(f"✓ Retrieved batch: {retrieved_batch.id}")
|
||||
print(f" Status: {retrieved_batch.status}")
|
||||
print(f" Output file: {retrieved_batch.output_file_id}")
|
||||
assert retrieved_batch.id == batch_obj.id
|
||||
assert retrieved_batch.object == "batch"
|
||||
assert retrieved_batch.input_file_id == file_obj.id
|
||||
|
||||
# Step 4: Retrieve file (verify file still accessible)
|
||||
print("\n=== Step 4: Retrieving original file ===")
|
||||
retrieved_file = await litellm.afile_retrieve(
|
||||
file_id=file_obj.id,
|
||||
custom_llm_provider="hosted_vllm",
|
||||
api_base=SERVER_URL,
|
||||
api_key="test-api-key",
|
||||
)
|
||||
|
||||
print(f"✓ Retrieved file: {retrieved_file.id}")
|
||||
print(f" Filename: {retrieved_file.filename}")
|
||||
print(f" Bytes: {retrieved_file.bytes}")
|
||||
assert retrieved_file.id == file_obj.id
|
||||
assert retrieved_file.object == "file"
|
||||
|
||||
print("\n✅ Full workflow test completed successfully!")
|
||||
Loading…
Add table
Reference in a new issue