diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 819a279a43c..246ac4fd369 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -706,6 +706,10 @@ def _get_batch_job_usage_from_response_body( if ResponseAPILoggingUtils._is_response_api_usage(_usage_dict): return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_usage_dict) usage: Final[Usage] = Usage(**_usage_dict) + if custom_llm_provider == "xai": + from litellm.llms.xai.chat.transformation import XAIChatConfig + + XAIChatConfig.fold_reasoning_tokens_into_completion(usage) return usage diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 76b6c73b375..f977fc03891 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -31,6 +31,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.openai import OpenAIBatchesAPI from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction +from litellm.llms.xai.batches.handler import XAIBatchesHandler from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( CancelBatchRequest, @@ -59,6 +60,7 @@ openai_batches_instance: Final = OpenAIBatchesAPI() azure_batches_instance: Final = AzureBatchesAPI() vertex_ai_batches_instance: Final = VertexAIBatchPrediction(gcs_bucket_name="") anthropic_batches_instance: Final = AnthropicBatchesHandler() +xai_batches_instance: Final = XAIBatchesHandler() base_llm_http_handler = BaseLLMHTTPHandler() ################################################# @@ -105,10 +107,22 @@ def _resolve_timeout( @client async def acreate_batch( completion_window: Literal["24h"], - endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"], + endpoint: Literal[ + "/v1/chat/completions", + "/v1/embeddings", + "/v1/completions", + "/v1/responses", + "/v1/ocr", + "/v1/images/generations", + "/v1/images/edits", + "/v1/videos/generations", + "/v1/videos", + "/v1/videos/edits", + "/v1/videos/extensions", + ], input_file_id: str, custom_llm_provider: Literal[ - "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral" + "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral", "xai" ] = "openai", metadata: dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, @@ -157,10 +171,22 @@ async def acreate_batch( @client def create_batch( completion_window: Literal["24h"], - endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"], + endpoint: Literal[ + "/v1/chat/completions", + "/v1/embeddings", + "/v1/completions", + "/v1/responses", + "/v1/ocr", + "/v1/images/generations", + "/v1/images/edits", + "/v1/videos/generations", + "/v1/videos", + "/v1/videos/edits", + "/v1/videos/extensions", + ], input_file_id: str, custom_llm_provider: Literal[ - "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral" + "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral", "xai" ] = "openai", metadata: dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, @@ -243,6 +269,14 @@ def create_batch( model=model, ) return response + if custom_llm_provider == LlmProviders.XAI.value: + return xai_batches_instance.create_batch( + _is_async=_is_async, + create_batch_data=_create_batch_request, + api_base=optional_params.api_base, + api_key=optional_params.api_key, + timeout=timeout, + ) api_base: str | None = None if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -345,7 +379,7 @@ def create_batch( async def aretrieve_batch( batch_id: str, custom_llm_provider: Literal[ - "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral" + "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral", "xai" ] = "openai", metadata: dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, @@ -393,10 +427,18 @@ def _handle_retrieve_batch_providers_without_provider_config( _retrieve_batch_request: RetrieveBatchRequest, _is_async: bool, custom_llm_provider: Literal[ - "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral" + "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral", "xai" ] = "openai", logging_obj: LiteLLMLoggingObj | None = None, ): + if custom_llm_provider == LlmProviders.XAI.value: + return xai_batches_instance.retrieve_batch( + _is_async=_is_async, + batch_id=batch_id, + api_base=optional_params.api_base, + api_key=optional_params.api_key, + timeout=timeout, + ) api_base: str | None = None if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -518,7 +560,7 @@ def _handle_retrieve_batch_providers_without_provider_config( def retrieve_batch( batch_id: str, custom_llm_provider: Literal[ - "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral" + "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral", "xai" ] = "openai", metadata: dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, @@ -741,6 +783,15 @@ def list_batches( timeout = 600.0 _is_async: Final = kwargs.pop("alist_batches", False) is True + if custom_llm_provider == LlmProviders.XAI.value: + return xai_batches_instance.list_batches( + _is_async=_is_async, + api_base=optional_params.api_base, + api_key=optional_params.api_key, + timeout=timeout, + after=after, + limit=limit, + ) if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( @@ -837,7 +888,7 @@ def list_batches( async def acancel_batch( batch_id: str, model: str | None = None, - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy"] = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy", "xai"] = "openai", metadata: dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, extra_body: dict[str, str] | None = None, @@ -883,7 +934,7 @@ async def acancel_batch( def cancel_batch( batch_id: str, model: str | None = None, - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy"] | str = "openai", + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy", "xai"] | str = "openai", metadata: dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, extra_body: dict[str, str] | None = None, @@ -933,6 +984,14 @@ def cancel_batch( ) _is_async: Final = kwargs.pop("acancel_batch", False) is True + if custom_llm_provider == LlmProviders.XAI.value: + return xai_batches_instance.cancel_batch( + _is_async=_is_async, + batch_id=batch_id, + api_base=optional_params.api_base, + api_key=optional_params.api_key, + timeout=timeout, + ) api_base: str | None = None if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: api_base = ( diff --git a/litellm/files/main.py b/litellm/files/main.py index 72832aeccc9..723784795b0 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -28,12 +28,15 @@ FileCreateProvider = Literal[ "manus", "anthropic", "mistral", + "xai", ] FileRetrieveProvider = Literal[ - "openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic", "mistral" + "openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic", "mistral", "xai" ] -FileDeleteProvider = Literal["openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic", "mistral"] -FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral"] +FileDeleteProvider = Literal[ + "openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic", "mistral", "xai" +] +FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral", "xai"] import litellm from litellm import get_secret_str from litellm.files.streaming import FileContentStreamingResponse @@ -49,6 +52,8 @@ from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.common_utils import get_openai_credentials from litellm.llms.openai.openai import FileDeleted, FileObject, OpenAIFilesAPI from litellm.llms.vertex_ai.files.handler import VertexAIFilesHandler +from litellm.llms.xai.batches.handler import XAIBatchesHandler +from litellm.llms.xai.batches.transformation import is_xai_batch_results_id from litellm.types.llms.openai import ( CreateFileRequest, FileContentRequest, @@ -103,6 +108,7 @@ openai_files_instance: Final = OpenAIFilesAPI() azure_files_instance: Final = AzureOpenAIFilesAPI() vertex_ai_files_instance: Final = VertexAIFilesHandler() bedrock_files_instance: Final = BedrockFilesHandler() +xai_batch_results_instance: Final = XAIBatchesHandler() ################################################# @@ -920,6 +926,15 @@ def file_content( client=client, ) + if custom_llm_provider == LlmProviders.XAI.value and is_xai_batch_results_id(file_id): + return xai_batch_results_instance.batch_results_content( + _is_async=_is_async, + batch_id=file_id, + api_base=optional_params.api_base, + api_key=optional_params.api_key, + timeout=timeout, + ) + # Check if provider has a custom files config (e.g., Anthropic, Manus) provider_config: Final = ProviderConfigManager.get_provider_files_config( model="", diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 22b8d850c83..a0f027cd58f 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -114,10 +114,9 @@ class HealthCheckHelpers: """ Health check for batch mode. - Calls list_batches for providers that support it (openai, hosted_vllm, azure, - vertex_ai). For all other providers (e.g. bedrock) the batch API surface doesn't - include list_batches, so we fall back to acompletion to verify connectivity and - credential validity instead. + Calls list_batches for providers that support it. For all other providers (e.g. bedrock) + the batch API surface doesn't include list_batches, so we fall back to acompletion to + verify connectivity and credential validity instead. """ import litellm @@ -132,10 +131,9 @@ class HealthCheckHelpers: litellm_params={"api_base": api_base} if api_base else None, ) - if custom_llm_provider in LIST_BATCHES_SUPPORTED_PROVIDERS: - return await litellm.alist_batches(**filtered_model_params) - else: + if custom_llm_provider not in LIST_BATCHES_SUPPORTED_PROVIDERS: return await litellm.acompletion(**model_params) + return await litellm.alist_batches(**{**filtered_model_params, "custom_llm_provider": custom_llm_provider}) @staticmethod async def _image_edit_health_check(edit_request: Callable[[], Awaitable["ImageResponse"]]) -> "ImageResponse": diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f8145cbdc7e..0ef5bbf807a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -379,9 +379,12 @@ _DEPLOYMENT_PRICING_KEYS: Final = ( "output_cost_per_token", "input_cost_per_token_batches", "output_cost_per_token_batches", + "input_cost_per_token_above_200k_tokens_batches", "input_cost_per_token_above_272k_tokens_batches", + "output_cost_per_token_above_200k_tokens_batches", "output_cost_per_token_above_272k_tokens_batches", "cache_read_input_token_cost_batches", + "cache_read_input_token_cost_above_200k_tokens_batches", "cache_read_input_token_cost_above_272k_tokens_batches", "cache_creation_input_token_cost_batches", "cache_creation_input_token_cost_above_272k_tokens_batches", diff --git a/litellm/llms/xai/batches/__init__.py b/litellm/llms/xai/batches/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/xai/batches/handler.py b/litellm/llms/xai/batches/handler.py new file mode 100644 index 00000000000..62db1c4833a --- /dev/null +++ b/litellm/llms/xai/batches/handler.py @@ -0,0 +1,195 @@ +from collections.abc import Coroutine +from itertools import chain +from typing import Final + +import httpx +from typing_extensions import NotRequired, ReadOnly, TypedDict + +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) +from litellm.types.llms.openai import CreateBatchRequest, HttpxBinaryResponseContent +from litellm.types.utils import LiteLLMBatch, LlmProviders + +from .transformation import ( + XAI_RESULTS_PAGE_SIZE, + OpenAIBatchListResponse, + XAIBatch, + XAIBatchList, + XAIBatchResult, + XAIBatchResultsPage, + get_xai_auth_headers, + raise_for_xai_status, + results_to_openai_jsonl, + to_create_batch_body, + to_litellm_batch, + to_openai_batch_list, + xai_batches_url, +) + +_JSONL_CONTENT_TYPE: Final = ("content-type", "application/jsonl") + + +class _PageParams(TypedDict): + limit: ReadOnly[int] + pagination_token: NotRequired[ReadOnly[str]] + + +def _results_params(after: str | None, limit: int | None) -> dict[str, object]: # mutable-ok: httpx params + if after is None: + return dict(_PageParams(limit=limit or XAI_RESULTS_PAGE_SIZE)) # mutable-ok: httpx params + return dict(_PageParams(limit=limit or XAI_RESULTS_PAGE_SIZE, pagination_token=after)) # mutable-ok: httpx params + + +def _flatten(pages: list[XAIBatchResultsPage]) -> tuple[XAIBatchResult, ...]: + return tuple(chain.from_iterable(page.results for page in pages)) + + +def _jsonl_response(url: str, results: tuple[XAIBatchResult, ...]) -> HttpxBinaryResponseContent: + return HttpxBinaryResponseContent( + response=httpx.Response( + status_code=200, + content=results_to_openai_jsonl(results), + headers=(_JSONL_CONTENT_TYPE,), + request=httpx.Request(method="GET", url=url), + ) + ) + + +class XAIBatchesHandler: + def __init__(self, sync_client: HTTPHandler | None = None, async_client: AsyncHTTPHandler | None = None) -> None: + self._sync_client = sync_client + self._async_client = async_client + + def _sync(self, timeout: float | httpx.Timeout) -> HTTPHandler: + return self._sync_client or HTTPHandler(timeout=timeout) + + def _async(self, timeout: float | httpx.Timeout) -> AsyncHTTPHandler: + return self._async_client or get_async_httpx_client( + llm_provider=LlmProviders.XAI, + params={"timeout": timeout}, # mutable-ok: get_async_httpx_client takes a dict + ) + + def create_batch( + self, + _is_async: bool, + create_batch_data: CreateBatchRequest, + api_base: str | None, + api_key: str | None, + timeout: float | httpx.Timeout, + ) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]: + url: Final = xai_batches_url(api_base) + headers: Final = get_xai_auth_headers(api_key=api_key) + body: Final = dict(to_create_batch_body(create_batch_data)) # mutable-ok: httpx json body + endpoint: Final = create_batch_data.get("endpoint") or "/v1/chat/completions" + if _is_async: + + async def _acreate() -> LiteLLMBatch: + response: Final = await self._async(timeout).post(url, json=body, headers=headers, timeout=timeout) + return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json()), endpoint) + + return _acreate() + response: Final = self._sync(timeout).post(url, json=body, headers=headers, timeout=timeout) + return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json()), endpoint) + + def retrieve_batch( + self, + _is_async: bool, + batch_id: str, + api_base: str | None, + api_key: str | None, + timeout: float | httpx.Timeout, + ) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]: + url: Final = xai_batches_url(api_base, batch_id) + headers: Final = get_xai_auth_headers(api_key=api_key) + if _is_async: + + async def _aretrieve() -> LiteLLMBatch: + response: Final = await self._async(timeout).get(url, headers=headers, timeout=timeout) + return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json())) + + return _aretrieve() + response: Final = self._sync(timeout).get(url, headers=headers, timeout=timeout) + return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json())) + + def cancel_batch( + self, + _is_async: bool, + batch_id: str, + api_base: str | None, + api_key: str | None, + timeout: float | httpx.Timeout, + ) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]: + url: Final = xai_batches_url(api_base, batch_id, suffix=":cancel") + headers: Final = get_xai_auth_headers(api_key=api_key) + if _is_async: + + async def _acancel() -> LiteLLMBatch: + response: Final = await self._async(timeout).post(url, headers=headers, timeout=timeout) + return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json())) + + return _acancel() + response: Final = self._sync(timeout).post(url, headers=headers, timeout=timeout) + return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json())) + + def list_batches( + self, + _is_async: bool, + api_base: str | None, + api_key: str | None, + timeout: float | httpx.Timeout, + after: str | None = None, + limit: int | None = None, + ) -> OpenAIBatchListResponse | Coroutine[None, None, OpenAIBatchListResponse]: + url: Final = xai_batches_url(api_base) + headers: Final = get_xai_auth_headers(api_key=api_key) + params: Final = _results_params(after, limit) + if _is_async: + + async def _alist() -> OpenAIBatchListResponse: + response: Final = await self._async(timeout).get(url, params=params, headers=headers, timeout=timeout) + return to_openai_batch_list(XAIBatchList.model_validate(raise_for_xai_status(response).json())) + + return _alist() + response: Final = self._sync(timeout).get(url, params=params, headers=headers, timeout=timeout) + return to_openai_batch_list(XAIBatchList.model_validate(raise_for_xai_status(response).json())) + + def batch_results_content( + self, + _is_async: bool, + batch_id: str, + api_base: str | None, + api_key: str | None, + timeout: float | httpx.Timeout, + ) -> HttpxBinaryResponseContent | Coroutine[None, None, HttpxBinaryResponseContent]: + url: Final = xai_batches_url(api_base, batch_id, suffix="/results") + headers: Final = get_xai_auth_headers(api_key=api_key) + if _is_async: + + async def _aresults() -> HttpxBinaryResponseContent: + client: Final = self._async(timeout) + + async def _page(after: str | None) -> XAIBatchResultsPage: + response: Final = await client.get( + url, params=_results_params(after, None), headers=headers, timeout=timeout + ) + return XAIBatchResultsPage.model_validate(raise_for_xai_status(response).json()) + + pages = [await _page(None)] # mutable-ok: page walk terminates on the cursor, not on a fixed count + while pages[-1].pagination_token and pages[-1].results: + pages.append(await _page(pages[-1].pagination_token)) + return _jsonl_response(url, _flatten(pages)) + + return _aresults() + client: Final = self._sync(timeout) + + def _page(after: str | None) -> XAIBatchResultsPage: + response: Final = client.get(url, params=_results_params(after, None), headers=headers, timeout=timeout) + return XAIBatchResultsPage.model_validate(raise_for_xai_status(response).json()) + + pages = [_page(None)] # mutable-ok: page walk terminates on the cursor, not on a fixed count + while pages[-1].pagination_token and pages[-1].results: + pages.append(_page(pages[-1].pagination_token)) + return _jsonl_response(url, _flatten(pages)) diff --git a/litellm/llms/xai/batches/transformation.py b/litellm/llms/xai/batches/transformation.py new file mode 100644 index 00000000000..8f305b8c203 --- /dev/null +++ b/litellm/llms/xai/batches/transformation.py @@ -0,0 +1,278 @@ +""" +xAI Batch API reference: https://docs.x.ai/developers/advanced-api-usage/batch-api + +xAI batches carry request counters, not a status, and no output file: results are paged from +``GET /v1/batches/{id}/results``, so LiteLLM hands back the batch id as ``output_file_id``. +""" + +import json +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +import httpx +from openai.types.batch import BatchRequestCounts +from openai.types.batch import Errors as BatchErrors +from openai.types.batch_error import BatchError +from pydantic import BaseModel, ConfigDict +from typing_extensions import NotRequired, ReadOnly, TypedDict + +from litellm.constants import XAI_API_BASE +from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.xai.common_utils import XAIModelInfo +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import CreateBatchRequest +from litellm.types.utils import LiteLLMBatch + +OpenAIBatchStatus: TypeAlias = Literal[ + "validating", "failed", "in_progress", "finalizing", "completed", "expired", "cancelling", "cancelled" +] + +XAI_BATCH_ID_PREFIX: Final = "batch_" +XAI_RESULTS_PAGE_SIZE: Final = 1000 +DEFAULT_BATCH_NAME: Final = "litellm-batch" +DEFAULT_BATCH_ENDPOINT: Final = "/v1/chat/completions" +_EMPTY_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + + +class XAIBatchesError(BaseLLMException): + pass + + +def xai_batches_error( + error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers +) -> XAIBatchesError: + return XAIBatchesError( + status_code=status_code, + message=error_message, + headers=headers if isinstance(headers, httpx.Headers) else httpx.Headers(tuple(headers.items())), + ) + + +def raise_for_xai_status(response: httpx.Response) -> httpx.Response: + if response.status_code >= 400: + raise xai_batches_error(response.text, response.status_code, response.headers) + return response + + +def get_xai_api_base(api_base: str | None) -> str: + resolved: Final = (api_base or get_secret_str("XAI_API_BASE") or XAI_API_BASE).rstrip("/") + return resolved.removesuffix("/v1") + + +def get_xai_auth_headers( + headers: Mapping[str, str] = _EMPTY_HEADERS, api_key: str | None = None +) -> dict[str, str]: # mutable-ok: BaseConfig.validate_environment contract returns dict + resolved_key: Final = XAIModelInfo.get_api_key(api_key) + if resolved_key is None: + raise xai_batches_error( + "Missing xAI API Key. Pass api_key, set litellm.xai_key or XAI_API_KEY", 401, _EMPTY_HEADERS + ) + return dict(headers, Authorization=f"Bearer {resolved_key}") # mutable-ok: BaseConfig contract returns dict + + +def xai_batches_url(api_base: str | None, batch_id: str | None = None, suffix: str = "") -> str: + base: Final = f"{get_xai_api_base(api_base)}/v1/batches" + if batch_id is None: + return base + return f"{base}/{encode_url_path_segment(batch_id, field_name='batch_id')}{suffix}" + + +def is_xai_batch_results_id(file_id: str) -> bool: + return file_id.startswith(XAI_BATCH_ID_PREFIX) + + +class XAICreateBatchRequest(TypedDict): + name: ReadOnly[str] + input_file_id: NotRequired[ReadOnly[str]] + + +class XAIBatchState(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + num_requests: int = 0 + num_pending: int = 0 + num_success: int = 0 + num_error: int = 0 + num_cancelled: int = 0 + + +class XAIBatch(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + batch_id: str + name: str = "" + create_time: str | None = None + expire_time: str | None = None + cancel_time: str | None = None + cancel_by_xai_message: str | None = None + state: XAIBatchState = XAIBatchState() + input_file_id: str | None = None + + +class XAIBatchList(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + batches: tuple[XAIBatch, ...] = () + pagination_token: str | None = None + + +class XAIBatchResultError(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + code: int | str | None = None + message: str = "" + + +class XAIBatchResultData(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + response: Mapping[str, Mapping[str, object]] | None = None + error: XAIBatchResultError | None = None + + +class XAIBatchResult(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + batch_request_id: str + batch_result: XAIBatchResultData = XAIBatchResultData() + + +class XAIBatchResultsPage(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + results: tuple[XAIBatchResult, ...] = () + pagination_token: str | None = None + + +def _to_unix_timestamp(value: str | None) -> int | None: + """xAI returns RFC 3339 timestamps over gRPC but a bare ``YYYY-MM-DD`` over REST.""" + if value is None: + return None + try: + parsed: Final = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return int((parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc)).timestamp()) + + +def xai_batch_status(batch: XAIBatch) -> OpenAIBatchStatus: + """xAI exposes counters, not a status. A batch xAI itself cancelled (input validation failed) is a failure, + a caller-cancelled batch is cancelled, an empty batch is still validating its input file, and a batch + with nothing pending has completed.""" + if batch.cancel_time is not None: + return "failed" if batch.cancel_by_xai_message else "cancelled" + if batch.state.num_requests == 0: + return "validating" + if batch.state.num_pending > 0: + return "in_progress" + return "completed" + + +def to_litellm_batch(batch: XAIBatch, endpoint: str = DEFAULT_BATCH_ENDPOINT) -> LiteLLMBatch: + status: Final = xai_batch_status(batch) + created_at: Final = _to_unix_timestamp(batch.create_time) + cancelled_at: Final = _to_unix_timestamp(batch.cancel_time) + errors: Final = ( + BatchErrors(object="list", data=[BatchError(message=batch.cancel_by_xai_message)]) # mutable-ok: openai type + if batch.cancel_by_xai_message + else None + ) + return LiteLLMBatch( + id=batch.batch_id, + object="batch", + endpoint=endpoint, + input_file_id=batch.input_file_id or "", + completion_window="24h", + status=status, + created_at=created_at if created_at is not None else 0, + expires_at=_to_unix_timestamp(batch.expire_time), + failed_at=cancelled_at if status == "failed" else None, + cancelled_at=cancelled_at if status == "cancelled" else None, + output_file_id=batch.batch_id if status == "completed" else None, + errors=errors, + request_counts=BatchRequestCounts( + total=batch.state.num_requests, + completed=batch.state.num_success, + failed=batch.state.num_error + batch.state.num_cancelled, + ), + metadata={"name": batch.name} if batch.name else None, # mutable-ok: LiteLLMBatch.metadata is a dict + ) + + +class OpenAIBatchListResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + object: Literal["list"] = "list" + data: tuple[LiteLLMBatch, ...] + first_id: str | None + last_id: str | None + has_more: bool + next_page_token: str | None = None + + +def to_openai_batch_list(page: XAIBatchList) -> OpenAIBatchListResponse: + data: Final = tuple(to_litellm_batch(b) for b in page.batches) + return OpenAIBatchListResponse( + data=data, + first_id=data[0].id if data else None, + last_id=data[-1].id if data else None, + has_more=bool(page.pagination_token), + next_page_token=page.pagination_token or None, + ) + + +def to_create_batch_body(create_batch_data: CreateBatchRequest) -> XAICreateBatchRequest: + input_file_id: Final = create_batch_data.get("input_file_id") + if not input_file_id: + raise xai_batches_error("input_file_id is required to create an xAI batch", 400, _EMPTY_HEADERS) + metadata: Final = create_batch_data.get("metadata") + name: Final = metadata.get("name") if metadata else None + return XAICreateBatchRequest(name=name or DEFAULT_BATCH_NAME, input_file_id=input_file_id) + + +class OpenAIBatchOutputError(TypedDict): + code: ReadOnly[str] + message: ReadOnly[str] + + +class OpenAIBatchOutputResponse(TypedDict): + status_code: ReadOnly[int] + request_id: ReadOnly[object] + body: ReadOnly[Mapping[str, object]] + + +class OpenAIBatchOutputLine(TypedDict): + id: ReadOnly[str] + custom_id: ReadOnly[str] + response: ReadOnly[OpenAIBatchOutputResponse | None] + error: ReadOnly[OpenAIBatchOutputError | None] + + +def _result_to_openai_line(result: XAIBatchResult) -> OpenAIBatchOutputLine: + """One output JSONL line. xAI wraps the body in a one-key map named after the endpoint + (``chat_get_completion``, ``responses``, ``image_generation``, ...); the value is the OpenAI body.""" + error: Final = result.batch_result.error + response: Final = result.batch_result.response + body: Final = next(iter(response.values()), None) if response else None + if body is None: + message: Final = error.message if error is not None else "xAI returned no response for this request" + code: Final = str(error.code) if error is not None and error.code is not None else "request_failed" + return OpenAIBatchOutputLine( + id=f"batch_req_{result.batch_request_id}", + custom_id=result.batch_request_id, + response=None, + error=OpenAIBatchOutputError(code=code, message=message), + ) + return OpenAIBatchOutputLine( + id=f"batch_req_{result.batch_request_id}", + custom_id=result.batch_request_id, + response=OpenAIBatchOutputResponse(status_code=200, request_id=body.get("id"), body=body), + error=None, + ) + + +def results_to_openai_jsonl(results: Sequence[XAIBatchResult]) -> bytes: + return "".join(f"{json.dumps(_result_to_openai_line(r), ensure_ascii=False)}\n" for r in results).encode() diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 33ee727dfab..e686d49e689 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -296,7 +296,7 @@ class XAIChatConfig(OpenAIGPTConfig): except Exception as e: verbose_logger.debug("Error extracting X.AI web search usage: %s", e) - self._fold_reasoning_tokens_into_completion(response) + self.fold_reasoning_tokens_into_completion(response) self._normalize_openai_compatible_usage_totals(getattr(response, "usage", None)) restated_usage: Final = _usage_restated_from_xai_ticks(getattr(response, "usage", None)) if restated_usage is not None: @@ -304,7 +304,7 @@ class XAIChatConfig(OpenAIGPTConfig): return response @staticmethod - def _fold_reasoning_tokens_into_completion( + def fold_reasoning_tokens_into_completion( target: ModelResponse | Usage | dict[str, Any] | None, ) -> None: """Reconcile xAI Usage to the OpenAI invariant. @@ -426,7 +426,7 @@ class XAIChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler): chunk["choices"] = [{"index": 0, "delta": {}, "finish_reason": None}] if "usage" in chunk and chunk["usage"] is not None: - XAIChatConfig._fold_reasoning_tokens_into_completion(chunk["usage"]) + XAIChatConfig.fold_reasoning_tokens_into_completion(chunk["usage"]) XAIChatConfig._normalize_openai_compatible_usage_totals(chunk["usage"]) parsed_chunk: Final = super().chunk_parser(chunk) diff --git a/litellm/llms/xai/files/__init__.py b/litellm/llms/xai/files/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/xai/files/transformation.py b/litellm/llms/xai/files/transformation.py new file mode 100644 index 00000000000..dbccca47b25 --- /dev/null +++ b/litellm/llms/xai/files/transformation.py @@ -0,0 +1,247 @@ +""" +xAI Files API reference: https://docs.x.ai/developers/rest-api-reference/inference/files + +xAI stores ``purpose`` as an empty string; LiteLLM reports uploads as ``batch``, the only purpose xAI files serve. +""" + +import time +from collections.abc import Mapping, Sequence +from typing import Final + +import httpx +from openai.types.file_deleted import FileDeleted +from pydantic import BaseModel, ConfigDict +from typing_extensions import ReadOnly, TypedDict + +from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj +from litellm.types.llms.openai import ( + CreateFileRequest, + FileContentRequest, + HttpxBinaryResponseContent, + OpenAICreateFileRequestOptionalParams, + OpenAIFileObject, + OpenAIFilesPurpose, +) +from litellm.types.utils import LlmProviders + +from ..batches.transformation import ( + get_xai_api_base, + get_xai_auth_headers, + raise_for_xai_status, + xai_batches_error, +) + +_NO_QUERY_PARAMS: Final[dict[str, str]] = {} # mutable-ok: BaseFilesConfig request transforms return tuple[str, dict] +_DEFAULT_PURPOSE: Final[OpenAIFilesPurpose] = "batch" + + +class XAIMultipartUpload(TypedDict): + file: ReadOnly[tuple[str, object, str]] + purpose: ReadOnly[tuple[None, str]] + + +class XAIFile(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + id: str + bytes: int = 0 + created_at: int | None = None + filename: str = "" + purpose: str = "" + expires_at: int | None = None + + +class XAIFileList(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + data: tuple[XAIFile, ...] = () + pagination_token: str | None = None + + +class XAIFileDeleted(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + id: str + deleted: bool = True + + +def _to_openai_file_object(file: XAIFile) -> OpenAIFileObject: + return OpenAIFileObject( + id=file.id, + bytes=file.bytes, + created_at=file.created_at if file.created_at is not None else int(time.time()), + filename=file.filename, + object="file", + purpose=_DEFAULT_PURPOSE, + status="uploaded", + expires_at=file.expires_at, + ) + + +def _api_base_from(litellm_params: Mapping[str, object]) -> str: + api_base: Final = litellm_params.get("api_base") + return get_xai_api_base(api_base if isinstance(api_base, str) else None) + + +class XAIFilesConfig(BaseFilesConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.XAI + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream: bool | None = None, + ) -> str: + return f"{get_xai_api_base(api_base)}/v1/files" + + def _file_url(self, file_id: str, litellm_params: Mapping[str, object], suffix: str = "") -> str: + encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id") + return f"{_api_base_from(litellm_params)}/v1/files/{encoded_file_id}{suffix}" + + def get_error_class( + self, error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers + ) -> BaseLLMException: + return xai_batches_error(error_message, status_code, headers) + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[object], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: BaseFilesConfig signature + return get_xai_auth_headers(headers, api_key) + + def get_supported_openai_params( + self, model: str + ) -> list[OpenAICreateFileRequestOptionalParams]: # mutable-ok: BaseFilesConfig signature + return ["purpose"] # mutable-ok: BaseFilesConfig signature + + def map_openai_params( + self, + non_default_params: Mapping[str, object], + optional_params: dict[str, object], # mutable-ok: BaseConfig signature, returned as-is + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: BaseConfig signature + return optional_params + + def transform_create_file_request( + self, + model: str, + create_file_data: CreateFileRequest, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> dict[str, object]: # mutable-ok: BaseFilesConfig signature + if "file" not in create_file_data: + raise ValueError("File data is required") + extracted: Final = extract_file_data(create_file_data["file"]) + filename: Final = extracted["filename"] or f"file_{int(time.time())}.jsonl" + content_type: Final = extracted.get("content_type") or "application/octet-stream" + upload: Final = XAIMultipartUpload( + file=(filename, extracted["content"], content_type), + purpose=(None, create_file_data.get("purpose") or _DEFAULT_PURPOSE), + ) + return dict(upload) # mutable-ok: BaseFilesConfig signature + + def transform_create_file_response( + self, + model: str | None, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + ) -> OpenAIFileObject: + return _to_openai_file_object(XAIFile.model_validate(raise_for_xai_status(raw_response).json())) + + def transform_retrieve_file_request( + self, + file_id: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature + return self._file_url(file_id, litellm_params), _NO_QUERY_PARAMS + + def transform_retrieve_file_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + ) -> OpenAIFileObject: + return _to_openai_file_object(XAIFile.model_validate(raise_for_xai_status(raw_response).json())) + + def transform_delete_file_request( + self, + file_id: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature + return self._file_url(file_id, litellm_params), _NO_QUERY_PARAMS + + def transform_delete_file_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + ) -> FileDeleted: + deleted: Final = XAIFileDeleted.model_validate(raise_for_xai_status(raw_response).json()) + return FileDeleted(id=deleted.id, deleted=deleted.deleted, object="file") + + def transform_list_files_request( + self, + purpose: str | None, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature + return f"{_api_base_from(litellm_params)}/v1/files", _NO_QUERY_PARAMS + + def transform_list_files_next_request( + self, + raw_response: httpx.Response, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> tuple[str, dict[str, str]] | None: # mutable-ok: BaseFilesConfig signature + page: Final = XAIFileList.model_validate(raw_response.json()) + if not page.pagination_token or not page.data: + return None + return f"{_api_base_from(litellm_params)}/v1/files", {"pagination_token": page.pagination_token} + + def transform_list_files_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + ) -> list[OpenAIFileObject]: # mutable-ok: BaseFilesConfig signature + return [ # mutable-ok: BaseFilesConfig signature + _to_openai_file_object(f) + for f in XAIFileList.model_validate(raise_for_xai_status(raw_response).json()).data + ] + + def transform_file_content_request( + self, + file_content_request: FileContentRequest, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature + file_id: Final = file_content_request.get("file_id") + if file_id is None: + raise ValueError("file_id is required to download file content") + return self._file_url(file_id, litellm_params, suffix="/content"), _NO_QUERY_PARAMS + + def transform_file_content_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + ) -> HttpxBinaryResponseContent: + return HttpxBinaryResponseContent(response=raw_response) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5a207dc4c02..a670200c132 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -51436,13 +51436,16 @@ }, "xai/grok-4.20-0309-reasoning": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, @@ -51450,8 +51453,11 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true @@ -51480,9 +51486,13 @@ "xai/grok-4.3": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, @@ -51490,6 +51500,8 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, @@ -51502,9 +51514,13 @@ "xai/grok-4.3-latest": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, @@ -51512,6 +51528,8 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, @@ -59483,13 +59501,16 @@ }, "xai/grok-4.20-0309-non-reasoning": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, @@ -59497,20 +59518,26 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-0309": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": false, "supports_prompt_caching": true, @@ -59519,8 +59546,11 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true, "supported_endpoints": [ @@ -62787,13 +62817,16 @@ }, "xai/grok-4.20": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, @@ -62801,21 +62834,27 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, "xai/grok-4.20-reasoning": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, @@ -62823,21 +62862,27 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, "xai/grok-4.20-reasoning-latest": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, @@ -62845,8 +62890,11 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true @@ -63067,13 +63115,16 @@ }, "xai/grok-4.20-non-reasoning": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, @@ -63081,20 +63132,26 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-non-reasoning-latest": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, @@ -63102,20 +63159,26 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" @@ -63127,20 +63190,26 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-latest": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" @@ -63152,8 +63221,11 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, @@ -75459,13 +75531,16 @@ }, "xai/grok-4.20-0309": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, @@ -75473,8 +75548,11 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 6e7e9da3498..99ab5920c4f 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -522,7 +522,19 @@ class CreateBatchRequest(TypedDict, total=False): """ completion_window: Literal["24h"] - endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"] + endpoint: Literal[ + "/v1/chat/completions", + "/v1/embeddings", + "/v1/completions", + "/v1/responses", + "/v1/ocr", + "/v1/images/generations", + "/v1/images/edits", + "/v1/videos/generations", + "/v1/videos", + "/v1/videos/edits", + "/v1/videos/extensions", + ] input_file_id: str metadata: dict[str, str] | None output_expires_after: FileExpiresAfter diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3e306b48887..cd336c9b989 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -299,6 +299,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_272k_tokens_flex: float | None cache_read_input_token_cost_above_512k_tokens: float | None cache_read_input_token_cost_batches: ReadOnly[float | None] + cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] cache_creation_input_token_cost_batches: ReadOnly[float | None] cache_creation_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] @@ -327,8 +328,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_second: float | None # for OpenAI Speech models input_cost_per_token_batches: float | None input_cost_per_video_token_batches: ReadOnly[float | None] + input_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None] input_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token_batches: float | None + output_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None] output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token: Required[float | None] output_cost_per_token_flex: float | None # OpenAI flex service tier pricing @@ -3729,6 +3732,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_read_input_token_cost_above_272k_tokens_priority: float | None = None cache_read_input_token_cost_above_272k_tokens_flex: float | None = None cache_read_input_token_cost_batches: float | None = None + cache_read_input_token_cost_above_200k_tokens_batches: float | None = None cache_read_input_token_cost_above_272k_tokens_batches: float | None = None cache_creation_input_token_cost_batches: float | None = None cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None @@ -3742,6 +3746,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): input_cost_per_token_above_200k_tokens_priority: float | None = None input_cost_per_token_above_272k_tokens_priority: float | None = None input_cost_per_token_above_272k_tokens_flex: float | None = None + input_cost_per_token_above_200k_tokens_batches: float | None = None input_cost_per_token_above_272k_tokens_batches: float | None = None input_cost_per_query: float | None = None input_cost_per_image: float | None = None @@ -3766,6 +3771,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_token_above_200k_tokens_priority: float | None = None output_cost_per_token_above_272k_tokens_priority: float | None = None output_cost_per_token_above_272k_tokens_flex: float | None = None + output_cost_per_token_above_200k_tokens_batches: float | None = None output_cost_per_token_above_272k_tokens_batches: float | None = None output_cost_per_character_above_128k_tokens: float | None = None output_cost_per_image: float | None = None @@ -4140,7 +4146,7 @@ FILE_CONTENT_STREAMING_PROVIDERS: Final[frozenset[str]] = frozenset( LITELLM_EXECUTED_BATCH_PROVIDERS: Final[frozenset[str]] = frozenset({LlmProviders.HOSTED_VLLM.value}) -ListBatchesSupportedProvider = Literal["openai", "azure", "hosted_vllm", "litellm_proxy", "vertex_ai"] +ListBatchesSupportedProvider = Literal["openai", "azure", "hosted_vllm", "litellm_proxy", "vertex_ai", "xai"] LIST_BATCHES_SUPPORTED_PROVIDERS: Final[frozenset[str]] = frozenset(get_args(ListBatchesSupportedProvider)) diff --git a/litellm/utils.py b/litellm/utils.py index 4ea0769ea11..be4388802f9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6160,6 +6160,9 @@ def _get_model_info_helper( cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None), cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None), cache_read_input_token_cost_batches=_model_info.get("cache_read_input_token_cost_batches"), + cache_read_input_token_cost_above_200k_tokens_batches=_model_info.get( + "cache_read_input_token_cost_above_200k_tokens_batches" + ), cache_read_input_token_cost_above_272k_tokens_batches=_model_info.get( "cache_read_input_token_cost_above_272k_tokens_batches" ), @@ -6197,10 +6200,16 @@ def _get_model_info_helper( input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None), input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"), input_cost_per_video_token_batches=_model_info.get("input_cost_per_video_token_batches", None), + input_cost_per_token_above_200k_tokens_batches=_model_info.get( + "input_cost_per_token_above_200k_tokens_batches" + ), input_cost_per_token_above_272k_tokens_batches=_model_info.get( "input_cost_per_token_above_272k_tokens_batches" ), output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"), + output_cost_per_token_above_200k_tokens_batches=_model_info.get( + "output_cost_per_token_above_200k_tokens_batches" + ), output_cost_per_token_above_272k_tokens_batches=_model_info.get( "output_cost_per_token_above_272k_tokens_batches" ), @@ -9357,6 +9366,10 @@ class ProviderConfigManager: from litellm.llms.mistral.files.transformation import MistralFilesConfig return MistralFilesConfig() + elif LlmProviders.XAI == provider: + from litellm.llms.xai.files.transformation import XAIFilesConfig + + return XAIFilesConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5a207dc4c02..a670200c132 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -51436,13 +51436,16 @@ }, "xai/grok-4.20-0309-reasoning": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, @@ -51450,8 +51453,11 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true @@ -51480,9 +51486,13 @@ "xai/grok-4.3": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, @@ -51490,6 +51500,8 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, @@ -51502,9 +51514,13 @@ "xai/grok-4.3-latest": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, @@ -51512,6 +51528,8 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, @@ -59483,13 +59501,16 @@ }, "xai/grok-4.20-0309-non-reasoning": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, @@ -59497,20 +59518,26 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-0309": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": false, "supports_prompt_caching": true, @@ -59519,8 +59546,11 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true, "supported_endpoints": [ @@ -62787,13 +62817,16 @@ }, "xai/grok-4.20": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, @@ -62801,21 +62834,27 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, "xai/grok-4.20-reasoning": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, @@ -62823,21 +62862,27 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, "xai/grok-4.20-reasoning-latest": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, @@ -62845,8 +62890,11 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true @@ -63067,13 +63115,16 @@ }, "xai/grok-4.20-non-reasoning": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, @@ -63081,20 +63132,26 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-non-reasoning-latest": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, @@ -63102,20 +63159,26 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" @@ -63127,20 +63190,26 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-latest": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" @@ -63152,8 +63221,11 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, @@ -75459,13 +75531,16 @@ }, "xai/grok-4.20-0309": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 2e-06, "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, @@ -75473,8 +75548,11 @@ "supports_vision": true, "supports_web_search": true, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, + "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07, "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 35624045fdf..fa1828c780a 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -170,6 +170,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_read_input_token_cost_above_200k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_read_input_token_cost_above_200k_tokens_priority": { "type": "number", "minimum": 0, @@ -351,6 +356,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "input_cost_per_token_above_200k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "input_cost_per_token_above_200k_tokens_priority": { "type": "number", "minimum": 0, @@ -708,6 +718,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "output_cost_per_token_above_200k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "output_cost_per_token_above_200k_tokens_priority": { "type": "number", "minimum": 0, diff --git a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py index 1cc96cb1256..c3478c0d5eb 100644 --- a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py @@ -364,6 +364,26 @@ async def test_batch_health_check_uses_alist_batches_for_supported_providers(): mock_alist.assert_called_once() +@pytest.mark.asyncio +async def test_batch_health_check_hands_the_resolved_provider_to_alist_batches(): + filtered_model_params: Final = { + "model": "xai/grok-4.3", + "api_key": "sk-test", + "litellm_metadata": {"tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME]}, + } + + with patch("litellm.alist_batches", new_callable=AsyncMock, return_value={}) as mock_alist: + await HealthCheckHelpers._batch_health_check( + custom_llm_provider="xai", + model_params={**filtered_model_params, "messages": []}, + filtered_model_params=filtered_model_params, + ) + + assert mock_alist.call_args.kwargs["custom_llm_provider"] == "xai" + assert mock_alist.call_args.kwargs["model"] == "xai/grok-4.3" + assert mock_alist.call_args.kwargs["api_key"] == "sk-test" + + @pytest.mark.asyncio async def test_batch_health_check_falls_back_to_acompletion_for_unsupported(): """Providers not in LIST_BATCHES_SUPPORTED_PROVIDERS fall back to acompletion.""" diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 166eeb53f5f..265fdb50836 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -7781,6 +7781,9 @@ _PUBLISHED_BATCH_RATES: Final = MappingProxyType( "output_cost_per_token_batches": 4.1e-6, "cache_read_input_token_cost_batches": 1.2e-7, "cache_creation_input_token_cost_batches": 1.3e-6, + "input_cost_per_token_above_200k_tokens_batches": 2.1e-6, + "output_cost_per_token_above_200k_tokens_batches": 5.1e-6, + "cache_read_input_token_cost_above_200k_tokens_batches": 2.2e-7, "input_cost_per_token_above_272k_tokens_batches": 3.1e-6, "output_cost_per_token_above_272k_tokens_batches": 7.1e-6, "cache_read_input_token_cost_above_272k_tokens_batches": 3.2e-7, @@ -7789,14 +7792,17 @@ _PUBLISHED_BATCH_RATES: Final = MappingProxyType( ) _PUBLISHED_INPUT_BATCH_KEYS: Final = ( "input_cost_per_token_batches", + "input_cost_per_token_above_200k_tokens_batches", "input_cost_per_token_above_272k_tokens_batches", "cache_read_input_token_cost_batches", + "cache_read_input_token_cost_above_200k_tokens_batches", "cache_read_input_token_cost_above_272k_tokens_batches", "cache_creation_input_token_cost_batches", "cache_creation_input_token_cost_above_272k_tokens_batches", ) _PUBLISHED_OUTPUT_BATCH_KEYS: Final = ( "output_cost_per_token_batches", + "output_cost_per_token_above_200k_tokens_batches", "output_cost_per_token_above_272k_tokens_batches", ) @@ -7885,22 +7891,22 @@ def test_batch_cost_calculator_bills_the_carried_output_tier_when_the_deployment ) +@pytest.mark.parametrize( + "tier_key", + ["input_cost_per_token_above_200k_tokens_batches", "input_cost_per_token_above_272k_tokens_batches"], +) def test_deployment_pricing_model_info_honors_a_tier_only_batch_override_over_the_published_flat_rates( - _published_batch_model: None, + _published_batch_model: None, tier_key: str ) -> None: from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info - info: Final = deployment_pricing_model_info( - _batch_deployment_id({"input_cost_per_token_above_272k_tokens_batches": 1e-3}), _PUBLISHED_BATCH_DEPLOYMENT - ) + info: Final = deployment_pricing_model_info(_batch_deployment_id({tier_key: 1e-3}), _PUBLISHED_BATCH_DEPLOYMENT) carried_keys: Final = tuple( - key - for key in (*_PUBLISHED_INPUT_BATCH_KEYS, *_PUBLISHED_OUTPUT_BATCH_KEYS) - if key != "input_cost_per_token_above_272k_tokens_batches" + key for key in (*_PUBLISHED_INPUT_BATCH_KEYS, *_PUBLISHED_OUTPUT_BATCH_KEYS) if key != tier_key ) assert info is not None - assert info["input_cost_per_token_above_272k_tokens_batches"] == 1e-3 + assert info[tier_key] == 1e-3 assert {key: info[key] for key in carried_keys} == {key: _PUBLISHED_BATCH_RATES[key] for key in carried_keys} diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index dd95addac40..b8b922f72a7 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -464,6 +464,40 @@ def test_total_cost_applies_the_long_context_batch_tier_per_line(): assert result.cost == pytest.approx((300_000 * 2e-6) + (10 * 6e-6) + (100 * 1e-6) + (10 * 4e-6)) +def test_xai_output_lines_bill_reasoning_tokens_as_completion_tokens(): + row = _success_row( + model="grok-4.3", + usage={ + "prompt_tokens": 615, + "completion_tokens": 3, + "total_tokens": 993, + "completion_tokens_details": {"reasoning_tokens": 375}, + }, + ) + + result = bu._aggregate_batch_cost_usage_models( + entries=[row], + custom_llm_provider="xai", + model_info=ModelInfo( + key="xai/grok-4.3", + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=1.25e-6, + output_cost_per_token=2.5e-6, + litellm_provider="xai", + mode="chat", + supported_openai_params=None, + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=2e-6, + ), + ) + + assert result.usage.completion_tokens == 378 + assert result.usage.total_tokens == 993 + assert result.cost == pytest.approx((615 * 1e-6) + (378 * 2e-6)) + + def test_total_usage_empty_is_zero(): result = bu._aggregate_batch_cost_usage_models(entries=[], custom_llm_provider="openai") assert result.cost == 0.0 diff --git a/tests/unit/llms/xai/batches/__init__.py b/tests/unit/llms/xai/batches/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/xai/batches/test_xai_batches_handler.py b/tests/unit/llms/xai/batches/test_xai_batches_handler.py new file mode 100644 index 00000000000..6dcdf06e7ab --- /dev/null +++ b/tests/unit/llms/xai/batches/test_xai_batches_handler.py @@ -0,0 +1,344 @@ +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.llms.xai.batches.transformation import XAIBatchesError +from litellm.types.utils import LiteLLMBatch + +API_BASE: Final = "https://api.x.ai" +KEY: Final = "xai-test-key" + + +@pytest.fixture(autouse=True) +def _httpx_transport_so_respx_can_intercept(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +_XAI_BATCH: Final = { + "batch_id": "batch_1", + "name": "litellm-batch", + "create_time": "2026-09-23", + "expire_time": "2026-10-23", + "cancel_time": None, + "cancel_by_xai_message": None, + "state": {"num_requests": 2, "num_pending": 0, "num_success": 2, "num_error": 0, "num_cancelled": 0}, + "input_file_id": "file_1", +} + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@respx.mock +async def test_create_batch_posts_input_file_id_with_bearer_auth(sync_mode: bool) -> None: + route: Final = respx.post(f"{API_BASE}/v1/batches").respond(200, json=_XAI_BATCH) + + kwargs: Final = { + "completion_window": "24h", + "endpoint": "/v1/embeddings", + "input_file_id": "file_1", + "custom_llm_provider": "xai", + "api_key": KEY, + "api_base": API_BASE, + } + batch: Final = litellm.create_batch(**kwargs) if sync_mode else await litellm.acreate_batch(**kwargs) + + assert isinstance(batch, LiteLLMBatch) + request: Final = route.calls.last.request + assert request.headers["authorization"] == f"Bearer {KEY}" + assert json.loads(request.content) == {"name": "litellm-batch", "input_file_id": "file_1"} + assert (batch.id, batch.endpoint, batch.status, batch.output_file_id) == ( + "batch_1", + "/v1/embeddings", + "completed", + "batch_1", + ) + + +@pytest.mark.parametrize( + "endpoint", + [ + "/v1/chat/completions", + "/v1/embeddings", + "/v1/completions", + "/v1/responses", + "/v1/ocr", + "/v1/images/generations", + "/v1/images/edits", + "/v1/videos/generations", + "/v1/videos", + "/v1/videos/edits", + "/v1/videos/extensions", + ], +) +@respx.mock +async def test_create_batch_keeps_image_and_video_endpoints_on_the_batch(endpoint: str) -> None: + respx.post(f"{API_BASE}/v1/batches").respond(200, json=_XAI_BATCH) + + batch: Final = await litellm.acreate_batch( + completion_window="24h", + endpoint=endpoint, + input_file_id="file_1", + custom_llm_provider="xai", + api_key=KEY, + api_base=API_BASE, + ) + + assert isinstance(batch, LiteLLMBatch) + assert batch.endpoint == endpoint + assert json.loads(respx.calls.last.request.content) == {"name": "litellm-batch", "input_file_id": "file_1"} + + +@respx.mock +async def test_retrieve_after_a_non_chat_create_reports_chat() -> None: + respx.post(f"{API_BASE}/v1/batches").respond(200, json=_XAI_BATCH) + respx.get(f"{API_BASE}/v1/batches/batch_1").respond(200, json=_XAI_BATCH) + + created: Final = await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/embeddings", + input_file_id="file_1", + custom_llm_provider="xai", + api_key=KEY, + api_base=API_BASE, + ) + retrieved: Final = await litellm.aretrieve_batch( + batch_id="batch_1", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE + ) + + assert isinstance(created, LiteLLMBatch) and isinstance(retrieved, LiteLLMBatch) + assert (created.endpoint, retrieved.endpoint) == ("/v1/embeddings", "/v1/chat/completions") + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@respx.mock +async def test_retrieve_batch_reads_native_batch_route(sync_mode: bool) -> None: + respx.get(f"{API_BASE}/v1/batches/batch_1").respond( + 200, json={**_XAI_BATCH, "state": {"num_requests": 2, "num_pending": 2}} + ) + + kwargs: Final = {"batch_id": "batch_1", "custom_llm_provider": "xai", "api_key": KEY, "api_base": API_BASE} + batch: Final = litellm.retrieve_batch(**kwargs) if sync_mode else await litellm.aretrieve_batch(**kwargs) + + assert isinstance(batch, LiteLLMBatch) + assert (batch.status, batch.output_file_id, batch.input_file_id, batch.endpoint) == ( + "in_progress", + None, + "file_1", + "/v1/chat/completions", + ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@respx.mock +async def test_cancel_batch_uses_colon_cancel_route(sync_mode: bool) -> None: + route: Final = respx.post(f"{API_BASE}/v1/batches/batch_1:cancel").respond( + 200, json={**_XAI_BATCH, "cancel_time": "2026-09-23", "state": {}} + ) + + kwargs: Final = {"batch_id": "batch_1", "custom_llm_provider": "xai", "api_key": KEY, "api_base": API_BASE} + batch: Final = litellm.cancel_batch(**kwargs) if sync_mode else await litellm.acancel_batch(**kwargs) + + assert route.called + assert isinstance(batch, LiteLLMBatch) + assert (batch.status, batch.endpoint) == ("cancelled", "/v1/chat/completions") + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@respx.mock +async def test_list_batches_forwards_cursor_and_returns_openai_list(sync_mode: bool) -> None: + route: Final = respx.get(f"{API_BASE}/v1/batches").respond( + 200, json={"batches": [_XAI_BATCH], "pagination_token": "next"} + ) + + kwargs: Final = {"custom_llm_provider": "xai", "api_key": KEY, "api_base": API_BASE, "after": "cur", "limit": 5} + listed: Final = litellm.list_batches(**kwargs) if sync_mode else await litellm.alist_batches(**kwargs) + + assert dict(route.calls.last.request.url.params) == {"limit": "5", "pagination_token": "cur"} + assert listed.object == "list" + assert [(b.id, b.endpoint) for b in listed.data] == [("batch_1", "/v1/chat/completions")] + assert (listed.has_more, listed.next_page_token) == (True, "next") + + +@respx.mock +async def test_list_batches_treats_empty_pagination_token_as_last_page() -> None: + respx.get(f"{API_BASE}/v1/batches").respond(200, json={"batches": [_XAI_BATCH], "pagination_token": ""}) + + listed: Final = await litellm.alist_batches(custom_llm_provider="xai", api_key=KEY, api_base=API_BASE) + + assert (listed.has_more, listed.next_page_token) == (False, None) + assert [batch.endpoint for batch in listed.data] == ["/v1/chat/completions"] + + +@respx.mock +async def test_file_content_stops_paging_on_empty_pagination_token() -> None: + route: Final = respx.get(f"{API_BASE}/v1/batches/batch_1/results").respond( + 200, + json={ + "results": [{"batch_request_id": "r1", "batch_result": {"error": {"code": 3, "message": "boom"}}}], + "pagination_token": "", + }, + ) + + content: Final = await litellm.afile_content( + file_id="batch_1", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE + ) + + assert route.call_count == 1 + assert len(content.content.decode().splitlines()) == 1 + + +@pytest.mark.parametrize("operation", ["create", "retrieve", "cancel", "list", "file_content"]) +@respx.mock +async def test_batch_calls_fall_back_to_litellm_xai_key(operation: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("XAI_API_KEY", raising=False) + monkeypatch.setattr(litellm, "xai_key", "configured-xai-key") + monkeypatch.setattr(litellm, "api_key", "generic-key-must-not-be-used") + routes: Final = { + "create": respx.post(f"{API_BASE}/v1/batches").respond(200, json=_XAI_BATCH), + "retrieve": respx.get(f"{API_BASE}/v1/batches/batch_1").respond(200, json=_XAI_BATCH), + "cancel": respx.post(f"{API_BASE}/v1/batches/batch_1:cancel").respond(200, json=_XAI_BATCH), + "list": respx.get(f"{API_BASE}/v1/batches").respond( + 200, json={"batches": [_XAI_BATCH], "pagination_token": None} + ), + "file_content": respx.get(f"{API_BASE}/v1/batches/batch_1/results").respond( + 200, json={"results": [], "pagination_token": None} + ), + } + + if operation == "create": + await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id="file_1", + custom_llm_provider="xai", + api_base=API_BASE, + ) + elif operation == "retrieve": + await litellm.aretrieve_batch(batch_id="batch_1", custom_llm_provider="xai", api_base=API_BASE) + elif operation == "cancel": + await litellm.acancel_batch(batch_id="batch_1", custom_llm_provider="xai", api_base=API_BASE) + elif operation == "list": + await litellm.alist_batches(custom_llm_provider="xai", api_base=API_BASE) + else: + await litellm.afile_content(file_id="batch_1", custom_llm_provider="xai", api_base=API_BASE) + + assert routes[operation].calls.last.request.headers["authorization"] == "Bearer configured-xai-key" + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@respx.mock +async def test_file_content_of_a_batch_id_walks_every_results_page(sync_mode: bool) -> None: + def _page(request: httpx.Request) -> httpx.Response: + token: Final = request.url.params.get("pagination_token") + if token is None: + return httpx.Response( + 200, + json={ + "results": [ + { + "batch_request_id": "r1", + "batch_result": {"response": {"chat_get_completion": {"id": "c1", "choices": []}}}, + } + ], + "pagination_token": "r1", + }, + ) + assert token == "r1" + return httpx.Response( + 200, + json={ + "results": [ + {"batch_request_id": "r2", "batch_result": {"error": {"code": 3, "message": "boom"}}}, + ], + "pagination_token": None, + }, + ) + + route: Final = respx.get(f"{API_BASE}/v1/batches/batch_1/results").mock(side_effect=_page) + + kwargs: Final = {"file_id": "batch_1", "custom_llm_provider": "xai", "api_key": KEY, "api_base": API_BASE} + content: Final = litellm.file_content(**kwargs) if sync_mode else await litellm.afile_content(**kwargs) + + assert route.call_count == 2 + assert [dict(c.request.url.params) for c in route.calls] == [ + {"limit": "1000"}, + {"limit": "1000", "pagination_token": "r1"}, + ] + assert [json.loads(line) for line in content.content.decode().splitlines()] == [ + { + "id": "batch_req_r1", + "custom_id": "r1", + "response": {"status_code": 200, "request_id": "c1", "body": {"id": "c1", "choices": []}}, + "error": None, + }, + {"id": "batch_req_r2", "custom_id": "r2", "response": None, "error": {"code": "3", "message": "boom"}}, + ] + + +@respx.mock +async def test_file_content_unwraps_image_and_video_result_bodies() -> None: + respx.get(f"{API_BASE}/v1/batches/batch_1/results").respond( + 200, + json={ + "results": [ + { + "batch_request_id": "img", + "batch_result": { + "response": {"image_generation": {"data": [{"url": "https://cdn.example/img.png"}]}} + }, + }, + { + "batch_request_id": "vid", + "batch_result": { + "response": {"video_generation": {"id": "vid_1", "url": "https://cdn.example/clip.mp4"}} + }, + }, + ], + "pagination_token": None, + }, + ) + + content: Final = await litellm.afile_content( + file_id="batch_1", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE + ) + + assert [json.loads(line)["response"]["body"] for line in content.content.decode().splitlines()] == [ + {"data": [{"url": "https://cdn.example/img.png"}]}, + {"id": "vid_1", "url": "https://cdn.example/clip.mp4"}, + ] + + +@respx.mock +async def test_missing_xai_key_is_a_401_before_any_request(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("XAI_API_KEY", raising=False) + monkeypatch.setattr(litellm, "xai_key", None) + monkeypatch.setattr(litellm, "api_key", "generic-key-must-not-be-used") + route: Final = respx.post(f"{API_BASE}/v1/batches").respond(200, json=_XAI_BATCH) + + with pytest.raises(XAIBatchesError) as exc: + await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id="file_1", + custom_llm_provider="xai", + api_base=API_BASE, + ) + + assert exc.value.status_code == 401 + assert route.called is False + + +@respx.mock +async def test_upstream_error_surfaces_status_code_and_body() -> None: + respx.get(f"{API_BASE}/v1/batches/batch_missing").respond(404, json={"code": "404", "error": "not found"}) + + with pytest.raises(XAIBatchesError) as exc: + await litellm.aretrieve_batch( + batch_id="batch_missing", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE + ) + + assert exc.value.status_code == 404 + assert "not found" in exc.value.message diff --git a/tests/unit/llms/xai/batches/test_xai_batches_transformation.py b/tests/unit/llms/xai/batches/test_xai_batches_transformation.py new file mode 100644 index 00000000000..5f2bb6a33ce --- /dev/null +++ b/tests/unit/llms/xai/batches/test_xai_batches_transformation.py @@ -0,0 +1,224 @@ +import json +from typing import Final + +import pytest + +from litellm.llms.xai.batches.transformation import ( + XAIBatch, + XAIBatchesError, + XAIBatchList, + XAIBatchResult, + XAIBatchResultsPage, + get_xai_api_base, + results_to_openai_jsonl, + to_create_batch_body, + to_litellm_batch, + to_openai_batch_list, + xai_batches_url, +) +from litellm.types.llms.openai import CreateBatchRequest + +SEPT_23_2026_UTC: Final = 1790121600 + + +def _xai_batch(**overrides: object) -> XAIBatch: + return XAIBatch.model_validate( + { + "batch_id": "batch_9bdf", + "name": "nightly", + "create_time": "2026-09-23", + "expire_time": "2026-10-23", + "cancel_time": None, + "cancel_by_xai_message": None, + "state": {"num_requests": 2, "num_pending": 0, "num_success": 2, "num_error": 0, "num_cancelled": 0}, + "input_file_id": "file_07", + **overrides, + } + ) + + +def test_completed_batch_exposes_batch_id_as_output_file_and_maps_counts() -> None: + batch: Final = to_litellm_batch(_xai_batch()) + + assert batch.model_dump(exclude_none=True) == { + "id": "batch_9bdf", + "object": "batch", + "endpoint": "/v1/chat/completions", + "input_file_id": "file_07", + "completion_window": "24h", + "status": "completed", + "created_at": SEPT_23_2026_UTC, + "expires_at": SEPT_23_2026_UTC + 30 * 86400, + "output_file_id": "batch_9bdf", + "request_counts": {"total": 2, "completed": 2, "failed": 0}, + "metadata": {"name": "nightly"}, + } + + +def test_pending_requests_mean_in_progress_and_no_output_file() -> None: + batch: Final = to_litellm_batch( + _xai_batch(state={"num_requests": 3, "num_pending": 1, "num_success": 1, "num_error": 1, "num_cancelled": 0}) + ) + + assert (batch.status, batch.output_file_id) == ("in_progress", None) + assert batch.request_counts is not None + assert batch.request_counts.model_dump() == {"total": 3, "completed": 1, "failed": 1} + + +def test_empty_batch_is_still_validating() -> None: + assert to_litellm_batch(_xai_batch(state={})).status == "validating" + + +def test_batch_cancelled_by_xai_validation_is_failed_with_the_message() -> None: + batch: Final = to_litellm_batch( + _xai_batch( + state={}, + cancel_time="2026-09-23T10:00:00Z", + cancel_by_xai_message="JSONL file validation failed: Model grok-nope is not supported", + ) + ) + + assert batch.status == "failed" + assert batch.failed_at == SEPT_23_2026_UTC + 10 * 3600 + assert batch.cancelled_at is None + assert batch.errors is not None and batch.errors.data is not None + assert [e.message for e in batch.errors.data] == ["JSONL file validation failed: Model grok-nope is not supported"] + + +def test_batch_cancelled_by_caller_is_cancelled() -> None: + batch: Final = to_litellm_batch(_xai_batch(cancel_time="2026-09-23")) + + assert (batch.status, batch.cancelled_at, batch.errors) == ("cancelled", SEPT_23_2026_UTC, None) + + +@pytest.mark.parametrize( + "endpoint", + [ + "/v1/images/generations", + "/v1/images/edits", + "/v1/videos/generations", + "/v1/videos/edits", + "/v1/videos/extensions", + ], +) +def test_create_body_accepts_image_and_video_endpoints(endpoint: str) -> None: + body: Final = to_create_batch_body( + CreateBatchRequest(completion_window="24h", endpoint=endpoint, input_file_id="file_07") + ) + + assert dict(body) == {"name": "litellm-batch", "input_file_id": "file_07"} + + +def test_create_body_uses_input_file_id_and_metadata_name() -> None: + body: Final = to_create_batch_body( + CreateBatchRequest( + completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file_07", metadata={"name": "n1"} + ) + ) + + assert dict(body) == {"name": "n1", "input_file_id": "file_07"} + + +def test_create_body_without_input_file_id_is_a_400() -> None: + with pytest.raises(XAIBatchesError) as exc: + to_create_batch_body(CreateBatchRequest(completion_window="24h", endpoint="/v1/chat/completions")) + + assert exc.value.status_code == 400 + + +def test_results_render_as_openai_output_jsonl_with_errors_per_line() -> None: + page: Final = XAIBatchResultsPage.model_validate( + { + "results": [ + { + "batch_request_id": "r1", + "batch_result": { + "response": { + "chat_get_completion": {"id": "c1", "object": "chat.completion", "choices": [], "usage": {}} + } + }, + }, + {"batch_request_id": "r2", "batch_result": {"error": {"code": 3, "message": "bad model"}}}, + {"batch_request_id": "r3", "batch_result": {}}, + ], + "pagination_token": None, + } + ) + + lines: Final = [json.loads(line) for line in results_to_openai_jsonl(page.results).decode().splitlines()] + + assert lines == [ + { + "id": "batch_req_r1", + "custom_id": "r1", + "response": { + "status_code": 200, + "request_id": "c1", + "body": {"id": "c1", "object": "chat.completion", "choices": [], "usage": {}}, + }, + "error": None, + }, + {"id": "batch_req_r2", "custom_id": "r2", "response": None, "error": {"code": "3", "message": "bad model"}}, + { + "id": "batch_req_r3", + "custom_id": "r3", + "response": None, + "error": {"code": "request_failed", "message": "xAI returned no response for this request"}, + }, + ] + + +@pytest.mark.parametrize( + ("response_key", "body"), + [ + ("responses", {"id": "resp_1", "output": []}), + ("image_generation", {"created": 1, "data": [{"url": "https://cdn.example/img.png"}]}), + ("video_generation", {"id": "vid_1", "url": "https://cdn.example/clip.mp4"}), + ], +) +def test_result_unwraps_the_single_response_key_into_the_openai_body( + response_key: str, body: dict[str, object] +) -> None: + result: Final = XAIBatchResult.model_validate( + {"batch_request_id": "r", "batch_result": {"response": {response_key: body}}} + ) + + line: Final = json.loads(results_to_openai_jsonl((result,)).decode()) + assert line["response"]["body"] == body + assert line["response"]["request_id"] == body.get("id") + assert response_key not in line["response"]["body"] + + +def test_retrieve_and_list_report_chat_because_xai_has_no_batch_endpoint() -> None: + retrieved: Final = to_litellm_batch(_xai_batch()) + listed: Final = to_openai_batch_list(XAIBatchList.model_validate({"batches": [_xai_batch().model_dump()]})) + + assert retrieved.endpoint == "/v1/chat/completions" + assert [batch.endpoint for batch in listed.data] == ["/v1/chat/completions"] + assert retrieved.metadata == {"name": "nightly"} + + +def test_list_page_maps_to_openai_list_with_cursor_flags() -> None: + page: Final = XAIBatchList.model_validate( + {"batches": [_xai_batch().model_dump(), _xai_batch(batch_id="batch_2").model_dump()], "pagination_token": "t"} + ) + + listed: Final = to_openai_batch_list(page) + + assert (listed.object, listed.first_id, listed.last_id, listed.has_more, listed.next_page_token) == ( + "list", + "batch_9bdf", + "batch_2", + True, + "t", + ) + assert [b.id for b in listed.data] == ["batch_9bdf", "batch_2"] + + +@pytest.mark.parametrize( + "api_base", ["https://api.x.ai", "https://api.x.ai/", "https://api.x.ai/v1", "https://api.x.ai/v1/"] +) +def test_api_base_never_doubles_the_v1_segment(api_base: str) -> None: + assert get_xai_api_base(api_base) == "https://api.x.ai" + assert xai_batches_url(api_base, "batch_1", ":cancel") == "https://api.x.ai/v1/batches/batch_1:cancel" + assert xai_batches_url(api_base) == "https://api.x.ai/v1/batches" diff --git a/tests/unit/llms/xai/files/__init__.py b/tests/unit/llms/xai/files/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/xai/files/test_xai_files_transformation.py b/tests/unit/llms/xai/files/test_xai_files_transformation.py new file mode 100644 index 00000000000..5a7d86bdfb7 --- /dev/null +++ b/tests/unit/llms/xai/files/test_xai_files_transformation.py @@ -0,0 +1,144 @@ +from typing import Final + +import httpx +import pytest +import respx +from pydantic import TypeAdapter + +import litellm +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.openai import OpenAIFileObject + +API_BASE: Final = "https://api.x.ai" +KEY: Final = "xai-test-key" + + +@pytest.fixture(autouse=True) +def _httpx_transport_so_respx_can_intercept(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +_XAI_FILE: Final = { + "bytes": 337, + "created_at": 1790197740, + "expires_at": None, + "filename": "batch.jsonl", + "id": "file_07", + "object": "file", + "purpose": "", +} + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@respx.mock +async def test_create_file_uploads_multipart_to_xai_and_reports_batch_purpose(sync_mode: bool) -> None: + route: Final = respx.post(f"{API_BASE}/v1/files").respond(200, json=_XAI_FILE) + + kwargs: Final = { + "file": ("batch.jsonl", b'{"custom_id":"r1"}\n', "application/jsonl"), + "purpose": "batch", + "custom_llm_provider": "xai", + "api_key": KEY, + "api_base": API_BASE, + } + created: Final = litellm.create_file(**kwargs) if sync_mode else await litellm.acreate_file(**kwargs) + + request: Final = route.calls.last.request + assert request.headers["authorization"] == f"Bearer {KEY}" + assert request.headers["content-type"].startswith("multipart/form-data") + assert b'filename="batch.jsonl"' in request.content + assert b'{"custom_id":"r1"}' in request.content + assert created.model_dump(exclude_none=True) == { + "id": "file_07", + "bytes": 337, + "created_at": 1790197740, + "filename": "batch.jsonl", + "object": "file", + "purpose": "batch", + "status": "uploaded", + } + + +@respx.mock +async def test_file_content_of_an_uploaded_file_downloads_original_bytes() -> None: + respx.get(f"{API_BASE}/v1/files/file_07/content").respond(200, content=b'{"custom_id":"r1"}\n') + + content: Final = await litellm.afile_content( + file_id="file_07", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE + ) + + assert content.content == b'{"custom_id":"r1"}\n' + + +@respx.mock +async def test_delete_file_maps_xai_deleted_object() -> None: + respx.delete(f"{API_BASE}/v1/files/file_07").respond(200, json={"id": "file_07", "deleted": True, "object": "file"}) + + deleted: Final = await litellm.afile_delete( + file_id="file_07", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE + ) + + assert deleted.model_dump() == {"id": "file_07", "deleted": True, "object": "file"} + + +@respx.mock +async def test_create_file_falls_back_to_litellm_xai_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("XAI_API_KEY", raising=False) + monkeypatch.setattr(litellm, "xai_key", "configured-xai-key") + monkeypatch.setattr(litellm, "api_key", "generic-key-must-not-be-used") + route: Final = respx.post(f"{API_BASE}/v1/files").respond(200, json=_XAI_FILE) + + await litellm.acreate_file( + file=("batch.jsonl", b'{"custom_id":"r1"}\n', "application/jsonl"), + purpose="batch", + custom_llm_provider="xai", + api_base=API_BASE, + ) + + assert route.calls.last.request.headers["authorization"] == "Bearer configured-xai-key" + + +@respx.mock +async def test_list_files_reads_data_array() -> None: + respx.get(f"{API_BASE}/v1/files").respond(200, json={"data": [_XAI_FILE], "pagination_token": None}) + + listed: Final = await litellm.afile_list(custom_llm_provider="xai", api_key=KEY, api_base=API_BASE) + + files: Final = TypeAdapter(tuple[OpenAIFileObject, ...]).validate_python(listed) + assert [f.id for f in files] == ["file_07"] + + +@respx.mock +async def test_list_files_walks_every_page_by_pagination_token() -> None: + route: Final = respx.get(f"{API_BASE}/v1/files").mock( + side_effect=[ + httpx.Response(200, json={"data": [_XAI_FILE], "pagination_token": "file_07"}), + httpx.Response(200, json={"data": [{**_XAI_FILE, "id": "file_08"}], "pagination_token": "file_08"}), + httpx.Response(200, json={"data": [], "pagination_token": "file_08"}), + ] + ) + + listed: Final = await litellm.afile_list(custom_llm_provider="xai", api_key=KEY, api_base=API_BASE) + + files: Final = TypeAdapter(tuple[OpenAIFileObject, ...]).validate_python(listed) + assert [f.id for f in files] == ["file_07", "file_08"] + assert [call.request.url.params.get("pagination_token") for call in route.calls] == [None, "file_07", "file_08"] + + +async def _retrieve_file(sync_mode: bool, file_id: str) -> None: + if sync_mode: + litellm.file_retrieve(file_id=file_id, custom_llm_provider="xai", api_key=KEY, api_base=API_BASE) + return + await litellm.afile_retrieve(file_id=file_id, custom_llm_provider="xai", api_key=KEY, api_base=API_BASE) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@respx.mock +async def test_retrieve_file_maps_xai_not_found_to_a_404_error(sync_mode: bool) -> None: + respx.get(f"{API_BASE}/v1/files/file_gone").respond(404, json={"code": "not-found", "error": "File not found"}) + + with pytest.raises(BaseLLMException) as raised: + await _retrieve_file(sync_mode, "file_gone") + + assert raised.value.status_code == 404 + assert "File not found" in str(raised.value) diff --git a/tests/unit/llms/xai/test_xai_chat_transformation.py b/tests/unit/llms/xai/test_xai_chat_transformation.py index 3fd666e4f50..704d0061103 100644 --- a/tests/unit/llms/xai/test_xai_chat_transformation.py +++ b/tests/unit/llms/xai/test_xai_chat_transformation.py @@ -16,7 +16,7 @@ from litellm.types.utils import ( class TestXAIReasoningTokenFolding: - """``_fold_reasoning_tokens_into_completion`` re-aligns xAI Usage to the OpenAI invariant.""" + """``fold_reasoning_tokens_into_completion`` re-aligns xAI Usage to the OpenAI invariant.""" @staticmethod def _make_response( @@ -45,7 +45,7 @@ class TestXAIReasoningTokenFolding: reasoning_tokens=312, ) - XAIChatConfig._fold_reasoning_tokens_into_completion(response) + XAIChatConfig.fold_reasoning_tokens_into_completion(response) usage = response.usage assert usage.completion_tokens == 322 @@ -59,7 +59,7 @@ class TestXAIReasoningTokenFolding: reasoning_tokens=312, ) - XAIChatConfig._fold_reasoning_tokens_into_completion(response) + XAIChatConfig.fold_reasoning_tokens_into_completion(response) assert response.usage.completion_tokens == 322 @@ -71,7 +71,7 @@ class TestXAIReasoningTokenFolding: reasoning_tokens=0, ) - XAIChatConfig._fold_reasoning_tokens_into_completion(response) + XAIChatConfig.fold_reasoning_tokens_into_completion(response) assert response.usage.completion_tokens == 10 @@ -84,7 +84,7 @@ class TestXAIReasoningTokenFolding: reasoning_tokens=312, ) - XAIChatConfig._fold_reasoning_tokens_into_completion(response) + XAIChatConfig.fold_reasoning_tokens_into_completion(response) assert response.usage.completion_tokens == 10 assert response.usage.total_tokens == 999 diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 99dea6366f9..62ef9f11c2e 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -4581,6 +4581,55 @@ def test_every_openai_entry_with_a_long_context_rate_and_a_batch_rate_declares_t assert undeclared == [] +@pytest.mark.parametrize("prefix", _BATCH_RATE_PREFIXES) +def test_every_xai_entry_with_a_long_context_rate_and_a_batch_rate_declares_the_batch_tier( + _local_model_cost_map: None, prefix: str +) -> None: + undeclared: Final = [ + name + for name, entry in litellm.model_cost.items() + if isinstance(entry, dict) + and entry.get("litellm_provider") == "xai" + and entry.get(f"{prefix}_above_200k_tokens") is not None + and entry.get(f"{prefix}_batches") is not None + and entry.get(f"{prefix}_above_200k_tokens_batches") is None + ] + + assert undeclared == [] + + +_XAI_TIERED_BATCH_MODEL: Final = "xai/grok-4.3" + + +def test_xai_batch_tier_discounts_the_long_context_rate_like_the_flat_batch_rate(_local_model_cost_map: None) -> None: + info: Final = litellm.get_model_info(_XAI_TIERED_BATCH_MODEL, custom_llm_provider="xai") + flat_discount: Final = info["input_cost_per_token_batches"] / info["input_cost_per_token"] + + for prefix in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost"): + tier_discount = info[f"{prefix}_above_200k_tokens_batches"] / info[f"{prefix}_above_200k_tokens"] + assert tier_discount == pytest.approx(flat_discount) + assert info[f"{prefix}_above_200k_tokens_batches"] < info[f"{prefix}_above_200k_tokens"] + + +@pytest.mark.parametrize( + ("prompt_tokens", "tier"), [(200_000, "_above_200k_tokens_batches"), (199_999, "_batches")] +) +def test_xai_batch_cost_calculator_bills_the_200k_batch_tier_inclusively( + _local_model_cost_map: None, prompt_tokens: int, tier: str +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + info: Final = litellm.get_model_info(_XAI_TIERED_BATCH_MODEL, custom_llm_provider="xai") + usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=64, total_tokens=prompt_tokens + 64) + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=usage, model=_XAI_TIERED_BATCH_MODEL, custom_llm_provider="xai" + ) + + assert prompt_cost == pytest.approx(prompt_tokens * info[f"input_cost_per_token{tier}"]) + assert completion_cost_value == pytest.approx(64 * info[f"output_cost_per_token{tier}"]) + + def test_batch_cost_calculator_ignores_malformed_batch_tier_keys(): from litellm.cost_calculator import batch_cost_calculator diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 768d8955b8e..0cdc52c9a93 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -774,6 +774,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_read_input_token_cost_above_32k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_128k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_200k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens_batches": {"type": "number"}, "cache_read_input_token_cost_above_256k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_272k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_272k_tokens_flex": {"type": "number"}, @@ -797,6 +798,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_video_token": {"type": "number"}, "input_cost_per_token_above_32k_tokens": {"type": "number"}, "input_cost_per_token_above_200k_tokens": {"type": "number"}, + "input_cost_per_token_above_200k_tokens_batches": {"type": "number"}, "input_cost_per_token_above_256k_tokens": {"type": "number"}, "input_cost_per_token_above_272k_tokens": {"type": "number"}, "input_cost_per_token_above_512k_tokens": {"type": "number"}, @@ -897,6 +899,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "output_cost_per_token_above_32k_tokens": {"type": "number"}, "output_cost_per_token_above_128k_tokens": {"type": "number"}, "output_cost_per_token_above_200k_tokens": {"type": "number"}, + "output_cost_per_token_above_200k_tokens_batches": {"type": "number"}, "output_cost_per_token_above_256k_tokens": {"type": "number"}, "output_cost_per_token_above_272k_tokens": {"type": "number"}, "output_cost_per_token_above_512k_tokens": {"type": "number"}, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index be7094f7f6c..1e0aed46923 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32981,6 +32981,8 @@ export interface components { cache_read_input_token_cost?: number | null; /** Cache Read Input Token Cost Above 200K Tokens */ cache_read_input_token_cost_above_200k_tokens?: number | null; + /** Cache Read Input Token Cost Above 200K Tokens Batches */ + cache_read_input_token_cost_above_200k_tokens_batches?: number | null; /** Cache Read Input Token Cost Above 200K Tokens Priority */ cache_read_input_token_cost_above_200k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 272K Tokens */ @@ -33059,6 +33061,8 @@ export interface components { input_cost_per_token_above_128k_tokens?: number | null; /** Input Cost Per Token Above 200K Tokens */ input_cost_per_token_above_200k_tokens?: number | null; + /** Input Cost Per Token Above 200K Tokens Batches */ + input_cost_per_token_above_200k_tokens_batches?: number | null; /** Input Cost Per Token Above 200K Tokens Priority */ input_cost_per_token_above_200k_tokens_priority?: number | null; /** Input Cost Per Token Above 272K Tokens */ @@ -33182,6 +33186,8 @@ export interface components { output_cost_per_token_above_128k_tokens?: number | null; /** Output Cost Per Token Above 200K Tokens */ output_cost_per_token_above_200k_tokens?: number | null; + /** Output Cost Per Token Above 200K Tokens Batches */ + output_cost_per_token_above_200k_tokens_batches?: number | null; /** Output Cost Per Token Above 200K Tokens Priority */ output_cost_per_token_above_200k_tokens_priority?: number | null; /** Output Cost Per Token Above 272K Tokens */ @@ -46768,6 +46774,8 @@ export interface components { cache_read_input_token_cost?: number | null; /** Cache Read Input Token Cost Above 200K Tokens */ cache_read_input_token_cost_above_200k_tokens?: number | null; + /** Cache Read Input Token Cost Above 200K Tokens Batches */ + cache_read_input_token_cost_above_200k_tokens_batches?: number | null; /** Cache Read Input Token Cost Above 200K Tokens Priority */ cache_read_input_token_cost_above_200k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 272K Tokens */ @@ -46846,6 +46854,8 @@ export interface components { input_cost_per_token_above_128k_tokens?: number | null; /** Input Cost Per Token Above 200K Tokens */ input_cost_per_token_above_200k_tokens?: number | null; + /** Input Cost Per Token Above 200K Tokens Batches */ + input_cost_per_token_above_200k_tokens_batches?: number | null; /** Input Cost Per Token Above 200K Tokens Priority */ input_cost_per_token_above_200k_tokens_priority?: number | null; /** Input Cost Per Token Above 272K Tokens */ @@ -46969,6 +46979,8 @@ export interface components { output_cost_per_token_above_128k_tokens?: number | null; /** Output Cost Per Token Above 200K Tokens */ output_cost_per_token_above_200k_tokens?: number | null; + /** Output Cost Per Token Above 200K Tokens Batches */ + output_cost_per_token_above_200k_tokens_batches?: number | null; /** Output Cost Per Token Above 200K Tokens Priority */ output_cost_per_token_above_200k_tokens_priority?: number | null; /** Output Cost Per Token Above 272K Tokens */