scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-26 01:12:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 98

feat(xai): add native xAI batches and files support (#42812)

Browse source 
* feat(xai): add native xAI batches and files support

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(xai): tighten batch handler typing and avoid Final redeclaration on star import

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(xai): walk batch result pages iteratively

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(xai): stop paging on empty pagination token and honor litellm.xai_key

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(xai): import NotRequired and TypedDict from typing_extensions for Python 3.10

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* feat(batches): accept image and video endpoints on batch create

* test(xai): lock batch endpoint, auth, and result contracts

The batches test package collided with litellm/batches under pytest prepend, so the All Other Providers shard could not collect the new tests.

* fix(xai): price grok batch usage at xAI's 20 percent batch discount

* fix(xai): map not-found file reads to 404, bill batch reasoning tokens, and add 200k batch tier rates

* refactor(xai): drop routine prose and move tests under tests/unit

* fix(health): hand the resolved provider to list_batches in batch-mode health checks

* test(xai): make tests/unit/llms/xai/batches a package

* fix(xai): walk every page of the files list by pagination_token

---------

Co-authored-by: Mubashir Osmani <mubashir@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: mubashir1osmani <mubashir.osmani777@gmail.com>
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-09-25 15:35:20 -07:00 • committed by GitHub
parent 8b68c3cd09
commit 8ef85a45ce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 1874 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/batches/batch_utils.py
View file

@ -706,6 +706,10 @@ def _get_batch_job_usage_from_response_body(
if ResponseAPILoggingUtils._is_response_api_usage(_usage_dict):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_usage_dict)
usage: Final[Usage] = Usage(**_usage_dict)
if custom_llm_provider == "xai":
from litellm.llms.xai.chat.transformation import XAIChatConfig
XAIChatConfig.fold_reasoning_tokens_into_completion(usage)
return usage

77
litellm/batches/main.py
View file

@ -31,6 +31,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.openai import OpenAIBatchesAPI
from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction
from litellm.llms.xai.batches.handler import XAIBatchesHandler
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
CancelBatchRequest,
@ -59,6 +60,7 @@ openai_batches_instance: Final = OpenAIBatchesAPI()
azure_batches_instance: Final = AzureBatchesAPI()
vertex_ai_batches_instance: Final = VertexAIBatchPrediction(gcs_bucket_name="")
anthropic_batches_instance: Final = AnthropicBatchesHandler()
xai_batches_instance: Final = XAIBatchesHandler()
base_llm_http_handler = BaseLLMHTTPHandler()
#################################################
@ -105,10 +107,22 @@ def _resolve_timeout(
@client
async def acreate_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"],
endpoint: Literal[
"/v1/chat/completions",
"/v1/embeddings",
"/v1/completions",
"/v1/responses",
"/v1/ocr",
"/v1/images/generations",
"/v1/images/edits",
"/v1/videos/generations",
"/v1/videos",
"/v1/videos/edits",
"/v1/videos/extensions",
],
input_file_id: str,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral", "xai"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
@ -157,10 +171,22 @@ async def acreate_batch(
@client
def create_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"],
endpoint: Literal[
"/v1/chat/completions",
"/v1/embeddings",
"/v1/completions",
"/v1/responses",
"/v1/ocr",
"/v1/images/generations",
"/v1/images/edits",
"/v1/videos/generations",
"/v1/videos",
"/v1/videos/edits",
"/v1/videos/extensions",
],
input_file_id: str,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral", "xai"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
@ -243,6 +269,14 @@ def create_batch(
model=model,
)
return response
if custom_llm_provider == LlmProviders.XAI.value:
return xai_batches_instance.create_batch(
_is_async=_is_async,
create_batch_data=_create_batch_request,
api_base=optional_params.api_base,
api_key=optional_params.api_key,
timeout=timeout,
)
api_base: str | None = None
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -345,7 +379,7 @@ def create_batch(
async def aretrieve_batch(
batch_id: str,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral", "xai"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
@ -393,10 +427,18 @@ def _handle_retrieve_batch_providers_without_provider_config(
_retrieve_batch_request: RetrieveBatchRequest,
_is_async: bool,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral", "xai"
] = "openai",
logging_obj: LiteLLMLoggingObj | None = None,
):
if custom_llm_provider == LlmProviders.XAI.value:
return xai_batches_instance.retrieve_batch(
_is_async=_is_async,
batch_id=batch_id,
api_base=optional_params.api_base,
api_key=optional_params.api_key,
timeout=timeout,
)
api_base: str | None = None
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -518,7 +560,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
def retrieve_batch(
batch_id: str,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral", "xai"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
@ -741,6 +783,15 @@ def list_batches(
timeout = 600.0
_is_async: Final = kwargs.pop("alist_batches", False) is True
if custom_llm_provider == LlmProviders.XAI.value:
return xai_batches_instance.list_batches(
_is_async=_is_async,
api_base=optional_params.api_base,
api_key=optional_params.api_key,
timeout=timeout,
after=after,
limit=limit,
)
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
@ -837,7 +888,7 @@ def list_batches(
async def acancel_batch(
batch_id: str,
model: str | None = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy", "xai"] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
extra_body: dict[str, str] | None = None,
@ -883,7 +934,7 @@ async def acancel_batch(
def cancel_batch(
batch_id: str,
model: str | None = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy"] | str = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy", "xai"] | str = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
extra_body: dict[str, str] | None = None,
@ -933,6 +984,14 @@ def cancel_batch(
)
_is_async: Final = kwargs.pop("acancel_batch", False) is True
if custom_llm_provider == LlmProviders.XAI.value:
return xai_batches_instance.cancel_batch(
_is_async=_is_async,
batch_id=batch_id,
api_base=optional_params.api_base,
api_key=optional_params.api_key,
timeout=timeout,
)
api_base: str | None = None
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
api_base = (

21
litellm/files/main.py
View file

@ -28,12 +28,15 @@ FileCreateProvider = Literal[
"manus",
"anthropic",
"mistral",
"xai",
]
FileRetrieveProvider = Literal[
"openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic", "mistral"
"openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic", "mistral", "xai"
]
FileDeleteProvider = Literal["openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic", "mistral"]
FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral"]
FileDeleteProvider = Literal[
"openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic", "mistral", "xai"
]
FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral", "xai"]
import litellm
from litellm import get_secret_str
from litellm.files.streaming import FileContentStreamingResponse
@ -49,6 +52,8 @@ from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai.openai import FileDeleted, FileObject, OpenAIFilesAPI
from litellm.llms.vertex_ai.files.handler import VertexAIFilesHandler
from litellm.llms.xai.batches.handler import XAIBatchesHandler
from litellm.llms.xai.batches.transformation import is_xai_batch_results_id
from litellm.types.llms.openai import (
CreateFileRequest,
FileContentRequest,
@ -103,6 +108,7 @@ openai_files_instance: Final = OpenAIFilesAPI()
azure_files_instance: Final = AzureOpenAIFilesAPI()
vertex_ai_files_instance: Final = VertexAIFilesHandler()
bedrock_files_instance: Final = BedrockFilesHandler()
xai_batch_results_instance: Final = XAIBatchesHandler()
#################################################
@ -920,6 +926,15 @@ def file_content(
client=client,
)
if custom_llm_provider == LlmProviders.XAI.value and is_xai_batch_results_id(file_id):
return xai_batch_results_instance.batch_results_content(
_is_async=_is_async,
batch_id=file_id,
api_base=optional_params.api_base,
api_key=optional_params.api_key,
timeout=timeout,
)
# Check if provider has a custom files config (e.g., Anthropic, Manus)
provider_config: Final = ProviderConfigManager.get_provider_files_config(
model="",

12
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -114,10 +114,9 @@ class HealthCheckHelpers:
"""
Health check for batch mode.
Calls list_batches for providers that support it (openai, hosted_vllm, azure,
vertex_ai). For all other providers (e.g. bedrock) the batch API surface doesn't
include list_batches, so we fall back to acompletion to verify connectivity and
credential validity instead.
Calls list_batches for providers that support it. For all other providers (e.g. bedrock)
the batch API surface doesn't include list_batches, so we fall back to acompletion to
verify connectivity and credential validity instead.
"""
import litellm
@ -132,10 +131,9 @@ class HealthCheckHelpers:
litellm_params={"api_base": api_base} if api_base else None,
)
if custom_llm_provider in LIST_BATCHES_SUPPORTED_PROVIDERS:
return await litellm.alist_batches(**filtered_model_params)
else:
if custom_llm_provider not in LIST_BATCHES_SUPPORTED_PROVIDERS:
return await litellm.acompletion(**model_params)
return await litellm.alist_batches(**{**filtered_model_params, "custom_llm_provider": custom_llm_provider})
@staticmethod
async def _image_edit_health_check(edit_request: Callable[[], Awaitable["ImageResponse"]]) -> "ImageResponse":

3
litellm/litellm_core_utils/litellm_logging.py
View file

@ -379,9 +379,12 @@ _DEPLOYMENT_PRICING_KEYS: Final = (
"output_cost_per_token",
"input_cost_per_token_batches",
"output_cost_per_token_batches",
"input_cost_per_token_above_200k_tokens_batches",
"input_cost_per_token_above_272k_tokens_batches",
"output_cost_per_token_above_200k_tokens_batches",
"output_cost_per_token_above_272k_tokens_batches",
"cache_read_input_token_cost_batches",
"cache_read_input_token_cost_above_200k_tokens_batches",
"cache_read_input_token_cost_above_272k_tokens_batches",
"cache_creation_input_token_cost_batches",
"cache_creation_input_token_cost_above_272k_tokens_batches",

0
litellm/llms/xai/batches/__init__.py Normal file
View file

195
litellm/llms/xai/batches/handler.py Normal file
View file

@ -0,0 +1,195 @@
from collections.abc import Coroutine
from itertools import chain
from typing import Final
import httpx
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
get_async_httpx_client,
)
from litellm.types.llms.openai import CreateBatchRequest, HttpxBinaryResponseContent
from litellm.types.utils import LiteLLMBatch, LlmProviders
from .transformation import (
XAI_RESULTS_PAGE_SIZE,
OpenAIBatchListResponse,
XAIBatch,
XAIBatchList,
XAIBatchResult,
XAIBatchResultsPage,
get_xai_auth_headers,
raise_for_xai_status,
results_to_openai_jsonl,
to_create_batch_body,
to_litellm_batch,
to_openai_batch_list,
xai_batches_url,
)
_JSONL_CONTENT_TYPE: Final = ("content-type", "application/jsonl")
class _PageParams(TypedDict):
limit: ReadOnly[int]
pagination_token: NotRequired[ReadOnly[str]]
def _results_params(after: str | None, limit: int | None) -> dict[str, object]: # mutable-ok: httpx params
if after is None:
return dict(_PageParams(limit=limit or XAI_RESULTS_PAGE_SIZE)) # mutable-ok: httpx params
return dict(_PageParams(limit=limit or XAI_RESULTS_PAGE_SIZE, pagination_token=after)) # mutable-ok: httpx params
def _flatten(pages: list[XAIBatchResultsPage]) -> tuple[XAIBatchResult, ...]:
return tuple(chain.from_iterable(page.results for page in pages))
def _jsonl_response(url: str, results: tuple[XAIBatchResult, ...]) -> HttpxBinaryResponseContent:
return HttpxBinaryResponseContent(
response=httpx.Response(
status_code=200,
content=results_to_openai_jsonl(results),
headers=(_JSONL_CONTENT_TYPE,),
request=httpx.Request(method="GET", url=url),
)
)
class XAIBatchesHandler:
def __init__(self, sync_client: HTTPHandler | None = None, async_client: AsyncHTTPHandler | None = None) -> None:
self._sync_client = sync_client
self._async_client = async_client
def _sync(self, timeout: float | httpx.Timeout) -> HTTPHandler:
return self._sync_client or HTTPHandler(timeout=timeout)
def _async(self, timeout: float | httpx.Timeout) -> AsyncHTTPHandler:
return self._async_client or get_async_httpx_client(
llm_provider=LlmProviders.XAI,
params={"timeout": timeout}, # mutable-ok: get_async_httpx_client takes a dict
)
def create_batch(
self,
_is_async: bool,
create_batch_data: CreateBatchRequest,
api_base: str | None,
api_key: str | None,
timeout: float | httpx.Timeout,
) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]:
url: Final = xai_batches_url(api_base)
headers: Final = get_xai_auth_headers(api_key=api_key)
body: Final = dict(to_create_batch_body(create_batch_data)) # mutable-ok: httpx json body
endpoint: Final = create_batch_data.get("endpoint") or "/v1/chat/completions"
if _is_async:
async def _acreate() -> LiteLLMBatch:
response: Final = await self._async(timeout).post(url, json=body, headers=headers, timeout=timeout)
return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json()), endpoint)
return _acreate()
response: Final = self._sync(timeout).post(url, json=body, headers=headers, timeout=timeout)
return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json()), endpoint)
def retrieve_batch(
self,
_is_async: bool,
batch_id: str,
api_base: str | None,
api_key: str | None,
timeout: float | httpx.Timeout,
) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]:
url: Final = xai_batches_url(api_base, batch_id)
headers: Final = get_xai_auth_headers(api_key=api_key)
if _is_async:
async def _aretrieve() -> LiteLLMBatch:
response: Final = await self._async(timeout).get(url, headers=headers, timeout=timeout)
return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json()))
return _aretrieve()
response: Final = self._sync(timeout).get(url, headers=headers, timeout=timeout)
return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json()))
def cancel_batch(
self,
_is_async: bool,
batch_id: str,
api_base: str | None,
api_key: str | None,
timeout: float | httpx.Timeout,
) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]:
url: Final = xai_batches_url(api_base, batch_id, suffix=":cancel")
headers: Final = get_xai_auth_headers(api_key=api_key)
if _is_async:
async def _acancel() -> LiteLLMBatch:
response: Final = await self._async(timeout).post(url, headers=headers, timeout=timeout)
return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json()))
return _acancel()
response: Final = self._sync(timeout).post(url, headers=headers, timeout=timeout)
return to_litellm_batch(XAIBatch.model_validate(raise_for_xai_status(response).json()))
def list_batches(
self,
_is_async: bool,
api_base: str | None,
api_key: str | None,
timeout: float | httpx.Timeout,
after: str | None = None,
limit: int | None = None,
) -> OpenAIBatchListResponse | Coroutine[None, None, OpenAIBatchListResponse]:
url: Final = xai_batches_url(api_base)
headers: Final = get_xai_auth_headers(api_key=api_key)
params: Final = _results_params(after, limit)
if _is_async:
async def _alist() -> OpenAIBatchListResponse:
response: Final = await self._async(timeout).get(url, params=params, headers=headers, timeout=timeout)
return to_openai_batch_list(XAIBatchList.model_validate(raise_for_xai_status(response).json()))
return _alist()
response: Final = self._sync(timeout).get(url, params=params, headers=headers, timeout=timeout)
return to_openai_batch_list(XAIBatchList.model_validate(raise_for_xai_status(response).json()))
def batch_results_content(
self,
_is_async: bool,
batch_id: str,
api_base: str | None,
api_key: str | None,
timeout: float | httpx.Timeout,
) -> HttpxBinaryResponseContent | Coroutine[None, None, HttpxBinaryResponseContent]:
url: Final = xai_batches_url(api_base, batch_id, suffix="/results")
headers: Final = get_xai_auth_headers(api_key=api_key)
if _is_async:
async def _aresults() -> HttpxBinaryResponseContent:
client: Final = self._async(timeout)
async def _page(after: str | None) -> XAIBatchResultsPage:
response: Final = await client.get(
url, params=_results_params(after, None), headers=headers, timeout=timeout
)
return XAIBatchResultsPage.model_validate(raise_for_xai_status(response).json())
pages = [await _page(None)] # mutable-ok: page walk terminates on the cursor, not on a fixed count
while pages[-1].pagination_token and pages[-1].results:
pages.append(await _page(pages[-1].pagination_token))
return _jsonl_response(url, _flatten(pages))
return _aresults()
client: Final = self._sync(timeout)
def _page(after: str | None) -> XAIBatchResultsPage:
response: Final = client.get(url, params=_results_params(after, None), headers=headers, timeout=timeout)
return XAIBatchResultsPage.model_validate(raise_for_xai_status(response).json())
pages = [_page(None)] # mutable-ok: page walk terminates on the cursor, not on a fixed count
while pages[-1].pagination_token and pages[-1].results:
pages.append(_page(pages[-1].pagination_token))
return _jsonl_response(url, _flatten(pages))

278
litellm/llms/xai/batches/transformation.py Normal file
View file

@ -0,0 +1,278 @@
"""
xAI Batch API reference: https://docs.x.ai/developers/advanced-api-usage/batch-api
xAI batches carry request counters, not a status, and no output file: results are paged from
``GET /v1/batches/{id}/results``, so LiteLLM hands back the batch id as ``output_file_id``.
"""
import json
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
import httpx
from openai.types.batch import BatchRequestCounts
from openai.types.batch import Errors as BatchErrors
from openai.types.batch_error import BatchError
from pydantic import BaseModel, ConfigDict
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.constants import XAI_API_BASE
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.xai.common_utils import XAIModelInfo
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import CreateBatchRequest
from litellm.types.utils import LiteLLMBatch
OpenAIBatchStatus: TypeAlias = Literal[
"validating", "failed", "in_progress", "finalizing", "completed", "expired", "cancelling", "cancelled"
]
XAI_BATCH_ID_PREFIX: Final = "batch_"
XAI_RESULTS_PAGE_SIZE: Final = 1000
DEFAULT_BATCH_NAME: Final = "litellm-batch"
DEFAULT_BATCH_ENDPOINT: Final = "/v1/chat/completions"
_EMPTY_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
class XAIBatchesError(BaseLLMException):
pass
def xai_batches_error(
error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers
) -> XAIBatchesError:
return XAIBatchesError(
status_code=status_code,
message=error_message,
headers=headers if isinstance(headers, httpx.Headers) else httpx.Headers(tuple(headers.items())),
)
def raise_for_xai_status(response: httpx.Response) -> httpx.Response:
if response.status_code >= 400:
raise xai_batches_error(response.text, response.status_code, response.headers)
return response
def get_xai_api_base(api_base: str | None) -> str:
resolved: Final = (api_base or get_secret_str("XAI_API_BASE") or XAI_API_BASE).rstrip("/")
return resolved.removesuffix("/v1")
def get_xai_auth_headers(
headers: Mapping[str, str] = _EMPTY_HEADERS, api_key: str | None = None
) -> dict[str, str]: # mutable-ok: BaseConfig.validate_environment contract returns dict
resolved_key: Final = XAIModelInfo.get_api_key(api_key)
if resolved_key is None:
raise xai_batches_error(
"Missing xAI API Key. Pass api_key, set litellm.xai_key or XAI_API_KEY", 401, _EMPTY_HEADERS
)
return dict(headers, Authorization=f"Bearer {resolved_key}") # mutable-ok: BaseConfig contract returns dict
def xai_batches_url(api_base: str | None, batch_id: str | None = None, suffix: str = "") -> str:
base: Final = f"{get_xai_api_base(api_base)}/v1/batches"
if batch_id is None:
return base
return f"{base}/{encode_url_path_segment(batch_id, field_name='batch_id')}{suffix}"
def is_xai_batch_results_id(file_id: str) -> bool:
return file_id.startswith(XAI_BATCH_ID_PREFIX)
class XAICreateBatchRequest(TypedDict):
name: ReadOnly[str]
input_file_id: NotRequired[ReadOnly[str]]
class XAIBatchState(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
num_requests: int = 0
num_pending: int = 0
num_success: int = 0
num_error: int = 0
num_cancelled: int = 0
class XAIBatch(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
batch_id: str
name: str = ""
create_time: str | None = None
expire_time: str | None = None
cancel_time: str | None = None
cancel_by_xai_message: str | None = None
state: XAIBatchState = XAIBatchState()
input_file_id: str | None = None
class XAIBatchList(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
batches: tuple[XAIBatch, ...] = ()
pagination_token: str | None = None
class XAIBatchResultError(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
code: int | str | None = None
message: str = ""
class XAIBatchResultData(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
response: Mapping[str, Mapping[str, object]] | None = None
error: XAIBatchResultError | None = None
class XAIBatchResult(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
batch_request_id: str
batch_result: XAIBatchResultData = XAIBatchResultData()
class XAIBatchResultsPage(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
results: tuple[XAIBatchResult, ...] = ()
pagination_token: str | None = None
def _to_unix_timestamp(value: str | None) -> int | None:
"""xAI returns RFC 3339 timestamps over gRPC but a bare ``YYYY-MM-DD`` over REST."""
if value is None:
return None
try:
parsed: Final = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
return int((parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc)).timestamp())
def xai_batch_status(batch: XAIBatch) -> OpenAIBatchStatus:
"""xAI exposes counters, not a status. A batch xAI itself cancelled (input validation failed) is a failure,
a caller-cancelled batch is cancelled, an empty batch is still validating its input file, and a batch
with nothing pending has completed."""
if batch.cancel_time is not None:
return "failed" if batch.cancel_by_xai_message else "cancelled"
if batch.state.num_requests == 0:
return "validating"
if batch.state.num_pending > 0:
return "in_progress"
return "completed"
def to_litellm_batch(batch: XAIBatch, endpoint: str = DEFAULT_BATCH_ENDPOINT) -> LiteLLMBatch:
status: Final = xai_batch_status(batch)
created_at: Final = _to_unix_timestamp(batch.create_time)
cancelled_at: Final = _to_unix_timestamp(batch.cancel_time)
errors: Final = (
BatchErrors(object="list", data=[BatchError(message=batch.cancel_by_xai_message)]) # mutable-ok: openai type
if batch.cancel_by_xai_message
else None
)
return LiteLLMBatch(
id=batch.batch_id,
object="batch",
endpoint=endpoint,
input_file_id=batch.input_file_id or "",
completion_window="24h",
status=status,
created_at=created_at if created_at is not None else 0,
expires_at=_to_unix_timestamp(batch.expire_time),
failed_at=cancelled_at if status == "failed" else None,
cancelled_at=cancelled_at if status == "cancelled" else None,
output_file_id=batch.batch_id if status == "completed" else None,
errors=errors,
request_counts=BatchRequestCounts(
total=batch.state.num_requests,
completed=batch.state.num_success,
failed=batch.state.num_error + batch.state.num_cancelled,
),
metadata={"name": batch.name} if batch.name else None, # mutable-ok: LiteLLMBatch.metadata is a dict
)
class OpenAIBatchListResponse(BaseModel):
model_config = ConfigDict(frozen=True)
object: Literal["list"] = "list"
data: tuple[LiteLLMBatch, ...]
first_id: str | None
last_id: str | None
has_more: bool
next_page_token: str | None = None
def to_openai_batch_list(page: XAIBatchList) -> OpenAIBatchListResponse:
data: Final = tuple(to_litellm_batch(b) for b in page.batches)
return OpenAIBatchListResponse(
data=data,
first_id=data[0].id if data else None,
last_id=data[-1].id if data else None,
has_more=bool(page.pagination_token),
next_page_token=page.pagination_token or None,
)
def to_create_batch_body(create_batch_data: CreateBatchRequest) -> XAICreateBatchRequest:
input_file_id: Final = create_batch_data.get("input_file_id")
if not input_file_id:
raise xai_batches_error("input_file_id is required to create an xAI batch", 400, _EMPTY_HEADERS)
metadata: Final = create_batch_data.get("metadata")
name: Final = metadata.get("name") if metadata else None
return XAICreateBatchRequest(name=name or DEFAULT_BATCH_NAME, input_file_id=input_file_id)
class OpenAIBatchOutputError(TypedDict):
code: ReadOnly[str]
message: ReadOnly[str]
class OpenAIBatchOutputResponse(TypedDict):
status_code: ReadOnly[int]
request_id: ReadOnly[object]
body: ReadOnly[Mapping[str, object]]
class OpenAIBatchOutputLine(TypedDict):
id: ReadOnly[str]
custom_id: ReadOnly[str]
response: ReadOnly[OpenAIBatchOutputResponse | None]
error: ReadOnly[OpenAIBatchOutputError | None]
def _result_to_openai_line(result: XAIBatchResult) -> OpenAIBatchOutputLine:
"""One output JSONL line. xAI wraps the body in a one-key map named after the endpoint
(``chat_get_completion``, ``responses``, ``image_generation``, ...); the value is the OpenAI body."""
error: Final = result.batch_result.error
response: Final = result.batch_result.response
body: Final = next(iter(response.values()), None) if response else None
if body is None:
message: Final = error.message if error is not None else "xAI returned no response for this request"
code: Final = str(error.code) if error is not None and error.code is not None else "request_failed"
return OpenAIBatchOutputLine(
id=f"batch_req_{result.batch_request_id}",
custom_id=result.batch_request_id,
response=None,
error=OpenAIBatchOutputError(code=code, message=message),
)
return OpenAIBatchOutputLine(
id=f"batch_req_{result.batch_request_id}",
custom_id=result.batch_request_id,
response=OpenAIBatchOutputResponse(status_code=200, request_id=body.get("id"), body=body),
error=None,
)
def results_to_openai_jsonl(results: Sequence[XAIBatchResult]) -> bytes:
return "".join(f"{json.dumps(_result_to_openai_line(r), ensure_ascii=False)}\n" for r in results).encode()

6
litellm/llms/xai/chat/transformation.py
View file

@ -296,7 +296,7 @@ class XAIChatConfig(OpenAIGPTConfig):
except Exception as e:
verbose_logger.debug("Error extracting X.AI web search usage: %s", e)
self._fold_reasoning_tokens_into_completion(response)
self.fold_reasoning_tokens_into_completion(response)
self._normalize_openai_compatible_usage_totals(getattr(response, "usage", None))
restated_usage: Final = _usage_restated_from_xai_ticks(getattr(response, "usage", None))
if restated_usage is not None:
@ -304,7 +304,7 @@ class XAIChatConfig(OpenAIGPTConfig):
return response
@staticmethod
def _fold_reasoning_tokens_into_completion(
def fold_reasoning_tokens_into_completion(
target: ModelResponse | Usage | dict[str, Any] | None,
) -> None:
"""Reconcile xAI Usage to the OpenAI invariant.
@ -426,7 +426,7 @@ class XAIChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler):
chunk["choices"] = [{"index": 0, "delta": {}, "finish_reason": None}]
if "usage" in chunk and chunk["usage"] is not None:
XAIChatConfig._fold_reasoning_tokens_into_completion(chunk["usage"])
XAIChatConfig.fold_reasoning_tokens_into_completion(chunk["usage"])
XAIChatConfig._normalize_openai_compatible_usage_totals(chunk["usage"])
parsed_chunk: Final = super().chunk_parser(chunk)

0
litellm/llms/xai/files/__init__.py Normal file
View file

247
litellm/llms/xai/files/transformation.py Normal file
View file

@ -0,0 +1,247 @@
"""
xAI Files API reference: https://docs.x.ai/developers/rest-api-reference/inference/files
xAI stores ``purpose`` as an empty string; LiteLLM reports uploads as ``batch``, the only purpose xAI files serve.
"""
import time
from collections.abc import Mapping, Sequence
from typing import Final
import httpx
from openai.types.file_deleted import FileDeleted
from pydantic import BaseModel, ConfigDict
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj
from litellm.types.llms.openai import (
CreateFileRequest,
FileContentRequest,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
OpenAIFilesPurpose,
)
from litellm.types.utils import LlmProviders
from ..batches.transformation import (
get_xai_api_base,
get_xai_auth_headers,
raise_for_xai_status,
xai_batches_error,
)
_NO_QUERY_PARAMS: Final[dict[str, str]] = {} # mutable-ok: BaseFilesConfig request transforms return tuple[str, dict]
_DEFAULT_PURPOSE: Final[OpenAIFilesPurpose] = "batch"
class XAIMultipartUpload(TypedDict):
file: ReadOnly[tuple[str, object, str]]
purpose: ReadOnly[tuple[None, str]]
class XAIFile(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
id: str
bytes: int = 0
created_at: int | None = None
filename: str = ""
purpose: str = ""
expires_at: int | None = None
class XAIFileList(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
data: tuple[XAIFile, ...] = ()
pagination_token: str | None = None
class XAIFileDeleted(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
id: str
deleted: bool = True
def _to_openai_file_object(file: XAIFile) -> OpenAIFileObject:
return OpenAIFileObject(
id=file.id,
bytes=file.bytes,
created_at=file.created_at if file.created_at is not None else int(time.time()),
filename=file.filename,
object="file",
purpose=_DEFAULT_PURPOSE,
status="uploaded",
expires_at=file.expires_at,
)
def _api_base_from(litellm_params: Mapping[str, object]) -> str:
api_base: Final = litellm_params.get("api_base")
return get_xai_api_base(api_base if isinstance(api_base, str) else None)
class XAIFilesConfig(BaseFilesConfig):
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.XAI
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
stream: bool | None = None,
) -> str:
return f"{get_xai_api_base(api_base)}/v1/files"
def _file_url(self, file_id: str, litellm_params: Mapping[str, object], suffix: str = "") -> str:
encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id")
return f"{_api_base_from(litellm_params)}/v1/files/{encoded_file_id}{suffix}"
def get_error_class(
self, error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers
) -> BaseLLMException:
return xai_batches_error(error_message, status_code, headers)
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[object],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: BaseFilesConfig signature
return get_xai_auth_headers(headers, api_key)
def get_supported_openai_params(
self, model: str
) -> list[OpenAICreateFileRequestOptionalParams]: # mutable-ok: BaseFilesConfig signature
return ["purpose"] # mutable-ok: BaseFilesConfig signature
def map_openai_params(
self,
non_default_params: Mapping[str, object],
optional_params: dict[str, object], # mutable-ok: BaseConfig signature, returned as-is
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: BaseConfig signature
return optional_params
def transform_create_file_request(
self,
model: str,
create_file_data: CreateFileRequest,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> dict[str, object]: # mutable-ok: BaseFilesConfig signature
if "file" not in create_file_data:
raise ValueError("File data is required")
extracted: Final = extract_file_data(create_file_data["file"])
filename: Final = extracted["filename"] or f"file_{int(time.time())}.jsonl"
content_type: Final = extracted.get("content_type") or "application/octet-stream"
upload: Final = XAIMultipartUpload(
file=(filename, extracted["content"], content_type),
purpose=(None, create_file_data.get("purpose") or _DEFAULT_PURPOSE),
)
return dict(upload) # mutable-ok: BaseFilesConfig signature
def transform_create_file_response(
self,
model: str | None,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
) -> OpenAIFileObject:
return _to_openai_file_object(XAIFile.model_validate(raise_for_xai_status(raw_response).json()))
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature
return self._file_url(file_id, litellm_params), _NO_QUERY_PARAMS
def transform_retrieve_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
) -> OpenAIFileObject:
return _to_openai_file_object(XAIFile.model_validate(raise_for_xai_status(raw_response).json()))
def transform_delete_file_request(
self,
file_id: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature
return self._file_url(file_id, litellm_params), _NO_QUERY_PARAMS
def transform_delete_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
) -> FileDeleted:
deleted: Final = XAIFileDeleted.model_validate(raise_for_xai_status(raw_response).json())
return FileDeleted(id=deleted.id, deleted=deleted.deleted, object="file")
def transform_list_files_request(
self,
purpose: str | None,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature
return f"{_api_base_from(litellm_params)}/v1/files", _NO_QUERY_PARAMS
def transform_list_files_next_request(
self,
raw_response: httpx.Response,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, str]] | None: # mutable-ok: BaseFilesConfig signature
page: Final = XAIFileList.model_validate(raw_response.json())
if not page.pagination_token or not page.data:
return None
return f"{_api_base_from(litellm_params)}/v1/files", {"pagination_token": page.pagination_token}
def transform_list_files_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
) -> list[OpenAIFileObject]: # mutable-ok: BaseFilesConfig signature
return [ # mutable-ok: BaseFilesConfig signature
_to_openai_file_object(f)
for f in XAIFileList.model_validate(raise_for_xai_status(raw_response).json()).data
]
def transform_file_content_request(
self,
file_content_request: FileContentRequest,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature
file_id: Final = file_content_request.get("file_id")
if file_id is None:
raise ValueError("file_id is required to download file content")
return self._file_url(file_id, litellm_params, suffix="/content"), _NO_QUERY_PARAMS
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
) -> HttpxBinaryResponseContent:
return HttpxBinaryResponseContent(response=raw_response)

78
litellm/model_prices_and_context_window_backup.json
View file

@ -51436,13 +51436,16 @@
},
"xai/grok-4.20-0309-reasoning": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_reasoning": true,
@ -51450,8 +51453,11 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_prompt_caching": true,
"supports_response_schema": true
@ -51480,9 +51486,13 @@
"xai/grok-4.3": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_image_token": 1.25e-06,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
@ -51490,6 +51500,8 @@
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -51502,9 +51514,13 @@
"xai/grok-4.3-latest": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_image_token": 1.25e-06,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
@ -51512,6 +51528,8 @@
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -59483,13 +59501,16 @@
},
"xai/grok-4.20-0309-non-reasoning": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -59497,20 +59518,26 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true
},
"xai/grok-4.20-multi-agent-0309": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "responses",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": false,
"supports_prompt_caching": true,
@ -59519,8 +59546,11 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true,
"supported_endpoints": [
@ -62787,13 +62817,16 @@
},
"xai/grok-4.20": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_reasoning": true,
@ -62801,21 +62834,27 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_prompt_caching": true,
"supports_response_schema": true
},
"xai/grok-4.20-reasoning": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_reasoning": true,
@ -62823,21 +62862,27 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_prompt_caching": true,
"supports_response_schema": true
},
"xai/grok-4.20-reasoning-latest": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_reasoning": true,
@ -62845,8 +62890,11 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_prompt_caching": true,
"supports_response_schema": true
@ -63067,13 +63115,16 @@
},
"xai/grok-4.20-non-reasoning": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -63081,20 +63132,26 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true
},
"xai/grok-4.20-non-reasoning-latest": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -63102,20 +63159,26 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true
},
"xai/grok-4.20-multi-agent": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "responses",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supported_endpoints": [
"/v1/responses"
@ -63127,20 +63190,26 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true
},
"xai/grok-4.20-multi-agent-latest": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "responses",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supported_endpoints": [
"/v1/responses"
@ -63152,8 +63221,11 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true
},
@ -75459,13 +75531,16 @@
},
"xai/grok-4.20-0309": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_reasoning": true,
@ -75473,8 +75548,11 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_prompt_caching": true,
"supports_response_schema": true

14
litellm/types/llms/openai.py
View file

@ -522,7 +522,19 @@ class CreateBatchRequest(TypedDict, total=False):
"""
completion_window: Literal["24h"]
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"]
endpoint: Literal[
"/v1/chat/completions",
"/v1/embeddings",
"/v1/completions",
"/v1/responses",
"/v1/ocr",
"/v1/images/generations",
"/v1/images/edits",
"/v1/videos/generations",
"/v1/videos",
"/v1/videos/edits",
"/v1/videos/extensions",
]
input_file_id: str
metadata: dict[str, str] | None
output_expires_after: FileExpiresAfter

8
litellm/types/utils.py
View file

@ -299,6 +299,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_read_input_token_cost_above_272k_tokens_flex: float | None
cache_read_input_token_cost_above_512k_tokens: float | None
cache_read_input_token_cost_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None]
cache_creation_input_token_cost_batches: ReadOnly[float | None]
cache_creation_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None]
@ -327,8 +328,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_second: float | None # for OpenAI Speech models
input_cost_per_token_batches: float | None
input_cost_per_video_token_batches: ReadOnly[float | None]
input_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None]
input_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None]
output_cost_per_token_batches: float | None
output_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None]
output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None]
output_cost_per_token: Required[float | None]
output_cost_per_token_flex: float | None # OpenAI flex service tier pricing
@ -3729,6 +3732,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
cache_read_input_token_cost_above_272k_tokens_priority: float | None = None
cache_read_input_token_cost_above_272k_tokens_flex: float | None = None
cache_read_input_token_cost_batches: float | None = None
cache_read_input_token_cost_above_200k_tokens_batches: float | None = None
cache_read_input_token_cost_above_272k_tokens_batches: float | None = None
cache_creation_input_token_cost_batches: float | None = None
cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None
@ -3742,6 +3746,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
input_cost_per_token_above_200k_tokens_priority: float | None = None
input_cost_per_token_above_272k_tokens_priority: float | None = None
input_cost_per_token_above_272k_tokens_flex: float | None = None
input_cost_per_token_above_200k_tokens_batches: float | None = None
input_cost_per_token_above_272k_tokens_batches: float | None = None
input_cost_per_query: float | None = None
input_cost_per_image: float | None = None
@ -3766,6 +3771,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
output_cost_per_token_above_200k_tokens_priority: float | None = None
output_cost_per_token_above_272k_tokens_priority: float | None = None
output_cost_per_token_above_272k_tokens_flex: float | None = None
output_cost_per_token_above_200k_tokens_batches: float | None = None
output_cost_per_token_above_272k_tokens_batches: float | None = None
output_cost_per_character_above_128k_tokens: float | None = None
output_cost_per_image: float | None = None
@ -4140,7 +4146,7 @@ FILE_CONTENT_STREAMING_PROVIDERS: Final[frozenset[str]] = frozenset(
LITELLM_EXECUTED_BATCH_PROVIDERS: Final[frozenset[str]] = frozenset({LlmProviders.HOSTED_VLLM.value})
ListBatchesSupportedProvider = Literal["openai", "azure", "hosted_vllm", "litellm_proxy", "vertex_ai"]
ListBatchesSupportedProvider = Literal["openai", "azure", "hosted_vllm", "litellm_proxy", "vertex_ai", "xai"]
LIST_BATCHES_SUPPORTED_PROVIDERS: Final[frozenset[str]] = frozenset(get_args(ListBatchesSupportedProvider))

13
litellm/utils.py
View file

@ -6160,6 +6160,9 @@ def _get_model_info_helper(
cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None),
cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None),
cache_read_input_token_cost_batches=_model_info.get("cache_read_input_token_cost_batches"),
cache_read_input_token_cost_above_200k_tokens_batches=_model_info.get(
"cache_read_input_token_cost_above_200k_tokens_batches"
),
cache_read_input_token_cost_above_272k_tokens_batches=_model_info.get(
"cache_read_input_token_cost_above_272k_tokens_batches"
),
@ -6197,10 +6200,16 @@ def _get_model_info_helper(
input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None),
input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"),
input_cost_per_video_token_batches=_model_info.get("input_cost_per_video_token_batches", None),
input_cost_per_token_above_200k_tokens_batches=_model_info.get(
"input_cost_per_token_above_200k_tokens_batches"
),
input_cost_per_token_above_272k_tokens_batches=_model_info.get(
"input_cost_per_token_above_272k_tokens_batches"
),
output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"),
output_cost_per_token_above_200k_tokens_batches=_model_info.get(
"output_cost_per_token_above_200k_tokens_batches"
),
output_cost_per_token_above_272k_tokens_batches=_model_info.get(
"output_cost_per_token_above_272k_tokens_batches"
),
@ -9357,6 +9366,10 @@ class ProviderConfigManager:
from litellm.llms.mistral.files.transformation import MistralFilesConfig
return MistralFilesConfig()
elif LlmProviders.XAI == provider:
from litellm.llms.xai.files.transformation import XAIFilesConfig
return XAIFilesConfig()
return None
@staticmethod

78
model_prices_and_context_window.json
View file

@ -51436,13 +51436,16 @@
},
"xai/grok-4.20-0309-reasoning": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_reasoning": true,
@ -51450,8 +51453,11 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_prompt_caching": true,
"supports_response_schema": true
@ -51480,9 +51486,13 @@
"xai/grok-4.3": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_image_token": 1.25e-06,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
@ -51490,6 +51500,8 @@
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -51502,9 +51514,13 @@
"xai/grok-4.3-latest": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_image_token": 1.25e-06,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
@ -51512,6 +51528,8 @@
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -59483,13 +59501,16 @@
},
"xai/grok-4.20-0309-non-reasoning": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -59497,20 +59518,26 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true
},
"xai/grok-4.20-multi-agent-0309": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "responses",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": false,
"supports_prompt_caching": true,
@ -59519,8 +59546,11 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true,
"supported_endpoints": [
@ -62787,13 +62817,16 @@
},
"xai/grok-4.20": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_reasoning": true,
@ -62801,21 +62834,27 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_prompt_caching": true,
"supports_response_schema": true
},
"xai/grok-4.20-reasoning": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_reasoning": true,
@ -62823,21 +62862,27 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_prompt_caching": true,
"supports_response_schema": true
},
"xai/grok-4.20-reasoning-latest": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_reasoning": true,
@ -62845,8 +62890,11 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_prompt_caching": true,
"supports_response_schema": true
@ -63067,13 +63115,16 @@
},
"xai/grok-4.20-non-reasoning": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -63081,20 +63132,26 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true
},
"xai/grok-4.20-non-reasoning-latest": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -63102,20 +63159,26 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true
},
"xai/grok-4.20-multi-agent": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "responses",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supported_endpoints": [
"/v1/responses"
@ -63127,20 +63190,26 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true
},
"xai/grok-4.20-multi-agent-latest": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "responses",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supported_endpoints": [
"/v1/responses"
@ -63152,8 +63221,11 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_response_schema": true
},
@ -75459,13 +75531,16 @@
},
"xai/grok-4.20-0309": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1.6e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 2e-06,
"source": "https://api.x.ai/v1/language-models",
"supports_function_calling": true,
"supports_reasoning": true,
@ -75473,8 +75548,11 @@
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"output_cost_per_token_above_200k_tokens_batches": 4e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07,
"input_cost_per_image_token": 1.25e-06,
"supports_prompt_caching": true,
"supports_response_schema": true

15
model_prices_and_context_window.schema.json
View file

@ -170,6 +170,11 @@
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_read_input_token_cost_above_200k_tokens_batches": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_read_input_token_cost_above_200k_tokens_priority": {
"type": "number",
"minimum": 0,
@ -351,6 +356,11 @@
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"input_cost_per_token_above_200k_tokens_batches": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"input_cost_per_token_above_200k_tokens_priority": {
"type": "number",
"minimum": 0,
@ -708,6 +718,11 @@
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"output_cost_per_token_above_200k_tokens_batches": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"output_cost_per_token_above_200k_tokens_priority": {
"type": "number",
"minimum": 0,

20
tests/test_litellm/litellm_core_utils/test_health_check_helpers.py
View file

@ -364,6 +364,26 @@ async def test_batch_health_check_uses_alist_batches_for_supported_providers():
mock_alist.assert_called_once()
@pytest.mark.asyncio
async def test_batch_health_check_hands_the_resolved_provider_to_alist_batches():
filtered_model_params: Final = {
"model": "xai/grok-4.3",
"api_key": "sk-test",
"litellm_metadata": {"tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME]},
}
with patch("litellm.alist_batches", new_callable=AsyncMock, return_value={}) as mock_alist:
await HealthCheckHelpers._batch_health_check(
custom_llm_provider="xai",
model_params={**filtered_model_params, "messages": []},
filtered_model_params=filtered_model_params,
)
assert mock_alist.call_args.kwargs["custom_llm_provider"] == "xai"
assert mock_alist.call_args.kwargs["model"] == "xai/grok-4.3"
assert mock_alist.call_args.kwargs["api_key"] == "sk-test"
@pytest.mark.asyncio
async def test_batch_health_check_falls_back_to_acompletion_for_unsupported():
"""Providers not in LIST_BATCHES_SUPPORTED_PROVIDERS fall back to acompletion."""

22
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -7781,6 +7781,9 @@ _PUBLISHED_BATCH_RATES: Final = MappingProxyType(
"output_cost_per_token_batches": 4.1e-6,
"cache_read_input_token_cost_batches": 1.2e-7,
"cache_creation_input_token_cost_batches": 1.3e-6,
"input_cost_per_token_above_200k_tokens_batches": 2.1e-6,
"output_cost_per_token_above_200k_tokens_batches": 5.1e-6,
"cache_read_input_token_cost_above_200k_tokens_batches": 2.2e-7,
"input_cost_per_token_above_272k_tokens_batches": 3.1e-6,
"output_cost_per_token_above_272k_tokens_batches": 7.1e-6,
"cache_read_input_token_cost_above_272k_tokens_batches": 3.2e-7,
@ -7789,14 +7792,17 @@ _PUBLISHED_BATCH_RATES: Final = MappingProxyType(
)
_PUBLISHED_INPUT_BATCH_KEYS: Final = (
"input_cost_per_token_batches",
"input_cost_per_token_above_200k_tokens_batches",
"input_cost_per_token_above_272k_tokens_batches",
"cache_read_input_token_cost_batches",
"cache_read_input_token_cost_above_200k_tokens_batches",
"cache_read_input_token_cost_above_272k_tokens_batches",
"cache_creation_input_token_cost_batches",
"cache_creation_input_token_cost_above_272k_tokens_batches",
)
_PUBLISHED_OUTPUT_BATCH_KEYS: Final = (
"output_cost_per_token_batches",
"output_cost_per_token_above_200k_tokens_batches",
"output_cost_per_token_above_272k_tokens_batches",
)
@ -7885,22 +7891,22 @@ def test_batch_cost_calculator_bills_the_carried_output_tier_when_the_deployment
)
@pytest.mark.parametrize(
"tier_key",
["input_cost_per_token_above_200k_tokens_batches", "input_cost_per_token_above_272k_tokens_batches"],
)
def test_deployment_pricing_model_info_honors_a_tier_only_batch_override_over_the_published_flat_rates(
_published_batch_model: None,
_published_batch_model: None, tier_key: str
) -> None:
from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info
info: Final = deployment_pricing_model_info(
_batch_deployment_id({"input_cost_per_token_above_272k_tokens_batches": 1e-3}), _PUBLISHED_BATCH_DEPLOYMENT
)
info: Final = deployment_pricing_model_info(_batch_deployment_id({tier_key: 1e-3}), _PUBLISHED_BATCH_DEPLOYMENT)
carried_keys: Final = tuple(
key
for key in (*_PUBLISHED_INPUT_BATCH_KEYS, *_PUBLISHED_OUTPUT_BATCH_KEYS)
if key != "input_cost_per_token_above_272k_tokens_batches"
key for key in (*_PUBLISHED_INPUT_BATCH_KEYS, *_PUBLISHED_OUTPUT_BATCH_KEYS) if key != tier_key
)
assert info is not None
assert info["input_cost_per_token_above_272k_tokens_batches"] == 1e-3
assert info[tier_key] == 1e-3
assert {key: info[key] for key in carried_keys} == {key: _PUBLISHED_BATCH_RATES[key] for key in carried_keys}

34
tests/unit/batches/test_batch_utils.py
View file

@ -464,6 +464,40 @@ def test_total_cost_applies_the_long_context_batch_tier_per_line():
assert result.cost == pytest.approx((300_000 * 2e-6) + (10 * 6e-6) + (100 * 1e-6) + (10 * 4e-6))
def test_xai_output_lines_bill_reasoning_tokens_as_completion_tokens():
row = _success_row(
model="grok-4.3",
usage={
"prompt_tokens": 615,
"completion_tokens": 3,
"total_tokens": 993,
"completion_tokens_details": {"reasoning_tokens": 375},
},
)
result = bu._aggregate_batch_cost_usage_models(
entries=[row],
custom_llm_provider="xai",
model_info=ModelInfo(
key="xai/grok-4.3",
max_tokens=None,
max_input_tokens=None,
max_output_tokens=None,
input_cost_per_token=1.25e-6,
output_cost_per_token=2.5e-6,
litellm_provider="xai",
mode="chat",
supported_openai_params=None,
input_cost_per_token_batches=1e-6,
output_cost_per_token_batches=2e-6,
),
)
assert result.usage.completion_tokens == 378
assert result.usage.total_tokens == 993
assert result.cost == pytest.approx((615 * 1e-6) + (378 * 2e-6))
def test_total_usage_empty_is_zero():
result = bu._aggregate_batch_cost_usage_models(entries=[], custom_llm_provider="openai")
assert result.cost == 0.0

0
tests/unit/llms/xai/batches/__init__.py Normal file
View file

344
tests/unit/llms/xai/batches/test_xai_batches_handler.py Normal file
View file

@ -0,0 +1,344 @@
import json
from typing import Final
import httpx
import pytest
import respx
import litellm
from litellm.llms.xai.batches.transformation import XAIBatchesError
from litellm.types.utils import LiteLLMBatch
API_BASE: Final = "https://api.x.ai"
KEY: Final = "xai-test-key"
@pytest.fixture(autouse=True)
def _httpx_transport_so_respx_can_intercept(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
_XAI_BATCH: Final = {
"batch_id": "batch_1",
"name": "litellm-batch",
"create_time": "2026-09-23",
"expire_time": "2026-10-23",
"cancel_time": None,
"cancel_by_xai_message": None,
"state": {"num_requests": 2, "num_pending": 0, "num_success": 2, "num_error": 0, "num_cancelled": 0},
"input_file_id": "file_1",
}
@pytest.mark.parametrize("sync_mode", [True, False])
@respx.mock
async def test_create_batch_posts_input_file_id_with_bearer_auth(sync_mode: bool) -> None:
route: Final = respx.post(f"{API_BASE}/v1/batches").respond(200, json=_XAI_BATCH)
kwargs: Final = {
"completion_window": "24h",
"endpoint": "/v1/embeddings",
"input_file_id": "file_1",
"custom_llm_provider": "xai",
"api_key": KEY,
"api_base": API_BASE,
}
batch: Final = litellm.create_batch(**kwargs) if sync_mode else await litellm.acreate_batch(**kwargs)
assert isinstance(batch, LiteLLMBatch)
request: Final = route.calls.last.request
assert request.headers["authorization"] == f"Bearer {KEY}"
assert json.loads(request.content) == {"name": "litellm-batch", "input_file_id": "file_1"}
assert (batch.id, batch.endpoint, batch.status, batch.output_file_id) == (
"batch_1",
"/v1/embeddings",
"completed",
"batch_1",
)
@pytest.mark.parametrize(
"endpoint",
[
"/v1/chat/completions",
"/v1/embeddings",
"/v1/completions",
"/v1/responses",
"/v1/ocr",
"/v1/images/generations",
"/v1/images/edits",
"/v1/videos/generations",
"/v1/videos",
"/v1/videos/edits",
"/v1/videos/extensions",
],
)
@respx.mock
async def test_create_batch_keeps_image_and_video_endpoints_on_the_batch(endpoint: str) -> None:
respx.post(f"{API_BASE}/v1/batches").respond(200, json=_XAI_BATCH)
batch: Final = await litellm.acreate_batch(
completion_window="24h",
endpoint=endpoint,
input_file_id="file_1",
custom_llm_provider="xai",
api_key=KEY,
api_base=API_BASE,
)
assert isinstance(batch, LiteLLMBatch)
assert batch.endpoint == endpoint
assert json.loads(respx.calls.last.request.content) == {"name": "litellm-batch", "input_file_id": "file_1"}
@respx.mock
async def test_retrieve_after_a_non_chat_create_reports_chat() -> None:
respx.post(f"{API_BASE}/v1/batches").respond(200, json=_XAI_BATCH)
respx.get(f"{API_BASE}/v1/batches/batch_1").respond(200, json=_XAI_BATCH)
created: Final = await litellm.acreate_batch(
completion_window="24h",
endpoint="/v1/embeddings",
input_file_id="file_1",
custom_llm_provider="xai",
api_key=KEY,
api_base=API_BASE,
)
retrieved: Final = await litellm.aretrieve_batch(
batch_id="batch_1", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE
)
assert isinstance(created, LiteLLMBatch) and isinstance(retrieved, LiteLLMBatch)
assert (created.endpoint, retrieved.endpoint) == ("/v1/embeddings", "/v1/chat/completions")
@pytest.mark.parametrize("sync_mode", [True, False])
@respx.mock
async def test_retrieve_batch_reads_native_batch_route(sync_mode: bool) -> None:
respx.get(f"{API_BASE}/v1/batches/batch_1").respond(
200, json={**_XAI_BATCH, "state": {"num_requests": 2, "num_pending": 2}}
)
kwargs: Final = {"batch_id": "batch_1", "custom_llm_provider": "xai", "api_key": KEY, "api_base": API_BASE}
batch: Final = litellm.retrieve_batch(**kwargs) if sync_mode else await litellm.aretrieve_batch(**kwargs)
assert isinstance(batch, LiteLLMBatch)
assert (batch.status, batch.output_file_id, batch.input_file_id, batch.endpoint) == (
"in_progress",
None,
"file_1",
"/v1/chat/completions",
)
@pytest.mark.parametrize("sync_mode", [True, False])
@respx.mock
async def test_cancel_batch_uses_colon_cancel_route(sync_mode: bool) -> None:
route: Final = respx.post(f"{API_BASE}/v1/batches/batch_1:cancel").respond(
200, json={**_XAI_BATCH, "cancel_time": "2026-09-23", "state": {}}
)
kwargs: Final = {"batch_id": "batch_1", "custom_llm_provider": "xai", "api_key": KEY, "api_base": API_BASE}
batch: Final = litellm.cancel_batch(**kwargs) if sync_mode else await litellm.acancel_batch(**kwargs)
assert route.called
assert isinstance(batch, LiteLLMBatch)
assert (batch.status, batch.endpoint) == ("cancelled", "/v1/chat/completions")
@pytest.mark.parametrize("sync_mode", [True, False])
@respx.mock
async def test_list_batches_forwards_cursor_and_returns_openai_list(sync_mode: bool) -> None:
route: Final = respx.get(f"{API_BASE}/v1/batches").respond(
200, json={"batches": [_XAI_BATCH], "pagination_token": "next"}
)
kwargs: Final = {"custom_llm_provider": "xai", "api_key": KEY, "api_base": API_BASE, "after": "cur", "limit": 5}
listed: Final = litellm.list_batches(**kwargs) if sync_mode else await litellm.alist_batches(**kwargs)
assert dict(route.calls.last.request.url.params) == {"limit": "5", "pagination_token": "cur"}
assert listed.object == "list"
assert [(b.id, b.endpoint) for b in listed.data] == [("batch_1", "/v1/chat/completions")]
assert (listed.has_more, listed.next_page_token) == (True, "next")
@respx.mock
async def test_list_batches_treats_empty_pagination_token_as_last_page() -> None:
respx.get(f"{API_BASE}/v1/batches").respond(200, json={"batches": [_XAI_BATCH], "pagination_token": ""})
listed: Final = await litellm.alist_batches(custom_llm_provider="xai", api_key=KEY, api_base=API_BASE)
assert (listed.has_more, listed.next_page_token) == (False, None)
assert [batch.endpoint for batch in listed.data] == ["/v1/chat/completions"]
@respx.mock
async def test_file_content_stops_paging_on_empty_pagination_token() -> None:
route: Final = respx.get(f"{API_BASE}/v1/batches/batch_1/results").respond(
200,
json={
"results": [{"batch_request_id": "r1", "batch_result": {"error": {"code": 3, "message": "boom"}}}],
"pagination_token": "",
},
)
content: Final = await litellm.afile_content(
file_id="batch_1", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE
)
assert route.call_count == 1
assert len(content.content.decode().splitlines()) == 1
@pytest.mark.parametrize("operation", ["create", "retrieve", "cancel", "list", "file_content"])
@respx.mock
async def test_batch_calls_fall_back_to_litellm_xai_key(operation: str, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("XAI_API_KEY", raising=False)
monkeypatch.setattr(litellm, "xai_key", "configured-xai-key")
monkeypatch.setattr(litellm, "api_key", "generic-key-must-not-be-used")
routes: Final = {
"create": respx.post(f"{API_BASE}/v1/batches").respond(200, json=_XAI_BATCH),
"retrieve": respx.get(f"{API_BASE}/v1/batches/batch_1").respond(200, json=_XAI_BATCH),
"cancel": respx.post(f"{API_BASE}/v1/batches/batch_1:cancel").respond(200, json=_XAI_BATCH),
"list": respx.get(f"{API_BASE}/v1/batches").respond(
200, json={"batches": [_XAI_BATCH], "pagination_token": None}
),
"file_content": respx.get(f"{API_BASE}/v1/batches/batch_1/results").respond(
200, json={"results": [], "pagination_token": None}
),
}
if operation == "create":
await litellm.acreate_batch(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file_1",
custom_llm_provider="xai",
api_base=API_BASE,
)
elif operation == "retrieve":
await litellm.aretrieve_batch(batch_id="batch_1", custom_llm_provider="xai", api_base=API_BASE)
elif operation == "cancel":
await litellm.acancel_batch(batch_id="batch_1", custom_llm_provider="xai", api_base=API_BASE)
elif operation == "list":
await litellm.alist_batches(custom_llm_provider="xai", api_base=API_BASE)
else:
await litellm.afile_content(file_id="batch_1", custom_llm_provider="xai", api_base=API_BASE)
assert routes[operation].calls.last.request.headers["authorization"] == "Bearer configured-xai-key"
@pytest.mark.parametrize("sync_mode", [True, False])
@respx.mock
async def test_file_content_of_a_batch_id_walks_every_results_page(sync_mode: bool) -> None:
def _page(request: httpx.Request) -> httpx.Response:
token: Final = request.url.params.get("pagination_token")
if token is None:
return httpx.Response(
200,
json={
"results": [
{
"batch_request_id": "r1",
"batch_result": {"response": {"chat_get_completion": {"id": "c1", "choices": []}}},
}
],
"pagination_token": "r1",
},
)
assert token == "r1"
return httpx.Response(
200,
json={
"results": [
{"batch_request_id": "r2", "batch_result": {"error": {"code": 3, "message": "boom"}}},
],
"pagination_token": None,
},
)
route: Final = respx.get(f"{API_BASE}/v1/batches/batch_1/results").mock(side_effect=_page)
kwargs: Final = {"file_id": "batch_1", "custom_llm_provider": "xai", "api_key": KEY, "api_base": API_BASE}
content: Final = litellm.file_content(**kwargs) if sync_mode else await litellm.afile_content(**kwargs)
assert route.call_count == 2
assert [dict(c.request.url.params) for c in route.calls] == [
{"limit": "1000"},
{"limit": "1000", "pagination_token": "r1"},
]
assert [json.loads(line) for line in content.content.decode().splitlines()] == [
{
"id": "batch_req_r1",
"custom_id": "r1",
"response": {"status_code": 200, "request_id": "c1", "body": {"id": "c1", "choices": []}},
"error": None,
},
{"id": "batch_req_r2", "custom_id": "r2", "response": None, "error": {"code": "3", "message": "boom"}},
]
@respx.mock
async def test_file_content_unwraps_image_and_video_result_bodies() -> None:
respx.get(f"{API_BASE}/v1/batches/batch_1/results").respond(
200,
json={
"results": [
{
"batch_request_id": "img",
"batch_result": {
"response": {"image_generation": {"data": [{"url": "https://cdn.example/img.png"}]}}
},
},
{
"batch_request_id": "vid",
"batch_result": {
"response": {"video_generation": {"id": "vid_1", "url": "https://cdn.example/clip.mp4"}}
},
},
],
"pagination_token": None,
},
)
content: Final = await litellm.afile_content(
file_id="batch_1", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE
)
assert [json.loads(line)["response"]["body"] for line in content.content.decode().splitlines()] == [
{"data": [{"url": "https://cdn.example/img.png"}]},
{"id": "vid_1", "url": "https://cdn.example/clip.mp4"},
]
@respx.mock
async def test_missing_xai_key_is_a_401_before_any_request(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("XAI_API_KEY", raising=False)
monkeypatch.setattr(litellm, "xai_key", None)
monkeypatch.setattr(litellm, "api_key", "generic-key-must-not-be-used")
route: Final = respx.post(f"{API_BASE}/v1/batches").respond(200, json=_XAI_BATCH)
with pytest.raises(XAIBatchesError) as exc:
await litellm.acreate_batch(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file_1",
custom_llm_provider="xai",
api_base=API_BASE,
)
assert exc.value.status_code == 401
assert route.called is False
@respx.mock
async def test_upstream_error_surfaces_status_code_and_body() -> None:
respx.get(f"{API_BASE}/v1/batches/batch_missing").respond(404, json={"code": "404", "error": "not found"})
with pytest.raises(XAIBatchesError) as exc:
await litellm.aretrieve_batch(
batch_id="batch_missing", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE
)
assert exc.value.status_code == 404
assert "not found" in exc.value.message

224
tests/unit/llms/xai/batches/test_xai_batches_transformation.py Normal file
View file

@ -0,0 +1,224 @@
import json
from typing import Final
import pytest
from litellm.llms.xai.batches.transformation import (
XAIBatch,
XAIBatchesError,
XAIBatchList,
XAIBatchResult,
XAIBatchResultsPage,
get_xai_api_base,
results_to_openai_jsonl,
to_create_batch_body,
to_litellm_batch,
to_openai_batch_list,
xai_batches_url,
)
from litellm.types.llms.openai import CreateBatchRequest
SEPT_23_2026_UTC: Final = 1790121600
def _xai_batch(**overrides: object) -> XAIBatch:
return XAIBatch.model_validate(
{
"batch_id": "batch_9bdf",
"name": "nightly",
"create_time": "2026-09-23",
"expire_time": "2026-10-23",
"cancel_time": None,
"cancel_by_xai_message": None,
"state": {"num_requests": 2, "num_pending": 0, "num_success": 2, "num_error": 0, "num_cancelled": 0},
"input_file_id": "file_07",
**overrides,
}
)
def test_completed_batch_exposes_batch_id_as_output_file_and_maps_counts() -> None:
batch: Final = to_litellm_batch(_xai_batch())
assert batch.model_dump(exclude_none=True) == {
"id": "batch_9bdf",
"object": "batch",
"endpoint": "/v1/chat/completions",
"input_file_id": "file_07",
"completion_window": "24h",
"status": "completed",
"created_at": SEPT_23_2026_UTC,
"expires_at": SEPT_23_2026_UTC + 30 * 86400,
"output_file_id": "batch_9bdf",
"request_counts": {"total": 2, "completed": 2, "failed": 0},
"metadata": {"name": "nightly"},
}
def test_pending_requests_mean_in_progress_and_no_output_file() -> None:
batch: Final = to_litellm_batch(
_xai_batch(state={"num_requests": 3, "num_pending": 1, "num_success": 1, "num_error": 1, "num_cancelled": 0})
)
assert (batch.status, batch.output_file_id) == ("in_progress", None)
assert batch.request_counts is not None
assert batch.request_counts.model_dump() == {"total": 3, "completed": 1, "failed": 1}
def test_empty_batch_is_still_validating() -> None:
assert to_litellm_batch(_xai_batch(state={})).status == "validating"
def test_batch_cancelled_by_xai_validation_is_failed_with_the_message() -> None:
batch: Final = to_litellm_batch(
_xai_batch(
state={},
cancel_time="2026-09-23T10:00:00Z",
cancel_by_xai_message="JSONL file validation failed: Model grok-nope is not supported",
)
)
assert batch.status == "failed"
assert batch.failed_at == SEPT_23_2026_UTC + 10 * 3600
assert batch.cancelled_at is None
assert batch.errors is not None and batch.errors.data is not None
assert [e.message for e in batch.errors.data] == ["JSONL file validation failed: Model grok-nope is not supported"]
def test_batch_cancelled_by_caller_is_cancelled() -> None:
batch: Final = to_litellm_batch(_xai_batch(cancel_time="2026-09-23"))
assert (batch.status, batch.cancelled_at, batch.errors) == ("cancelled", SEPT_23_2026_UTC, None)
@pytest.mark.parametrize(
"endpoint",
[
"/v1/images/generations",
"/v1/images/edits",
"/v1/videos/generations",
"/v1/videos/edits",
"/v1/videos/extensions",
],
)
def test_create_body_accepts_image_and_video_endpoints(endpoint: str) -> None:
body: Final = to_create_batch_body(
CreateBatchRequest(completion_window="24h", endpoint=endpoint, input_file_id="file_07")
)
assert dict(body) == {"name": "litellm-batch", "input_file_id": "file_07"}
def test_create_body_uses_input_file_id_and_metadata_name() -> None:
body: Final = to_create_batch_body(
CreateBatchRequest(
completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file_07", metadata={"name": "n1"}
)
)
assert dict(body) == {"name": "n1", "input_file_id": "file_07"}
def test_create_body_without_input_file_id_is_a_400() -> None:
with pytest.raises(XAIBatchesError) as exc:
to_create_batch_body(CreateBatchRequest(completion_window="24h", endpoint="/v1/chat/completions"))
assert exc.value.status_code == 400
def test_results_render_as_openai_output_jsonl_with_errors_per_line() -> None:
page: Final = XAIBatchResultsPage.model_validate(
{
"results": [
{
"batch_request_id": "r1",
"batch_result": {
"response": {
"chat_get_completion": {"id": "c1", "object": "chat.completion", "choices": [], "usage": {}}
}
},
},
{"batch_request_id": "r2", "batch_result": {"error": {"code": 3, "message": "bad model"}}},
{"batch_request_id": "r3", "batch_result": {}},
],
"pagination_token": None,
}
)
lines: Final = [json.loads(line) for line in results_to_openai_jsonl(page.results).decode().splitlines()]
assert lines == [
{
"id": "batch_req_r1",
"custom_id": "r1",
"response": {
"status_code": 200,
"request_id": "c1",
"body": {"id": "c1", "object": "chat.completion", "choices": [], "usage": {}},
},
"error": None,
},
{"id": "batch_req_r2", "custom_id": "r2", "response": None, "error": {"code": "3", "message": "bad model"}},
{
"id": "batch_req_r3",
"custom_id": "r3",
"response": None,
"error": {"code": "request_failed", "message": "xAI returned no response for this request"},
},
]
@pytest.mark.parametrize(
("response_key", "body"),
[
("responses", {"id": "resp_1", "output": []}),
("image_generation", {"created": 1, "data": [{"url": "https://cdn.example/img.png"}]}),
("video_generation", {"id": "vid_1", "url": "https://cdn.example/clip.mp4"}),
],
)
def test_result_unwraps_the_single_response_key_into_the_openai_body(
response_key: str, body: dict[str, object]
) -> None:
result: Final = XAIBatchResult.model_validate(
{"batch_request_id": "r", "batch_result": {"response": {response_key: body}}}
)
line: Final = json.loads(results_to_openai_jsonl((result,)).decode())
assert line["response"]["body"] == body
assert line["response"]["request_id"] == body.get("id")
assert response_key not in line["response"]["body"]
def test_retrieve_and_list_report_chat_because_xai_has_no_batch_endpoint() -> None:
retrieved: Final = to_litellm_batch(_xai_batch())
listed: Final = to_openai_batch_list(XAIBatchList.model_validate({"batches": [_xai_batch().model_dump()]}))
assert retrieved.endpoint == "/v1/chat/completions"
assert [batch.endpoint for batch in listed.data] == ["/v1/chat/completions"]
assert retrieved.metadata == {"name": "nightly"}
def test_list_page_maps_to_openai_list_with_cursor_flags() -> None:
page: Final = XAIBatchList.model_validate(
{"batches": [_xai_batch().model_dump(), _xai_batch(batch_id="batch_2").model_dump()], "pagination_token": "t"}
)
listed: Final = to_openai_batch_list(page)
assert (listed.object, listed.first_id, listed.last_id, listed.has_more, listed.next_page_token) == (
"list",
"batch_9bdf",
"batch_2",
True,
"t",
)
assert [b.id for b in listed.data] == ["batch_9bdf", "batch_2"]
@pytest.mark.parametrize(
"api_base", ["https://api.x.ai", "https://api.x.ai/", "https://api.x.ai/v1", "https://api.x.ai/v1/"]
)
def test_api_base_never_doubles_the_v1_segment(api_base: str) -> None:
assert get_xai_api_base(api_base) == "https://api.x.ai"
assert xai_batches_url(api_base, "batch_1", ":cancel") == "https://api.x.ai/v1/batches/batch_1:cancel"
assert xai_batches_url(api_base) == "https://api.x.ai/v1/batches"

0
tests/unit/llms/xai/files/__init__.py Normal file
View file

144
tests/unit/llms/xai/files/test_xai_files_transformation.py Normal file
View file

@ -0,0 +1,144 @@
from typing import Final
import httpx
import pytest
import respx
from pydantic import TypeAdapter
import litellm
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import OpenAIFileObject
API_BASE: Final = "https://api.x.ai"
KEY: Final = "xai-test-key"
@pytest.fixture(autouse=True)
def _httpx_transport_so_respx_can_intercept(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
_XAI_FILE: Final = {
"bytes": 337,
"created_at": 1790197740,
"expires_at": None,
"filename": "batch.jsonl",
"id": "file_07",
"object": "file",
"purpose": "",
}
@pytest.mark.parametrize("sync_mode", [True, False])
@respx.mock
async def test_create_file_uploads_multipart_to_xai_and_reports_batch_purpose(sync_mode: bool) -> None:
route: Final = respx.post(f"{API_BASE}/v1/files").respond(200, json=_XAI_FILE)
kwargs: Final = {
"file": ("batch.jsonl", b'{"custom_id":"r1"}\n', "application/jsonl"),
"purpose": "batch",
"custom_llm_provider": "xai",
"api_key": KEY,
"api_base": API_BASE,
}
created: Final = litellm.create_file(**kwargs) if sync_mode else await litellm.acreate_file(**kwargs)
request: Final = route.calls.last.request
assert request.headers["authorization"] == f"Bearer {KEY}"
assert request.headers["content-type"].startswith("multipart/form-data")
assert b'filename="batch.jsonl"' in request.content
assert b'{"custom_id":"r1"}' in request.content
assert created.model_dump(exclude_none=True) == {
"id": "file_07",
"bytes": 337,
"created_at": 1790197740,
"filename": "batch.jsonl",
"object": "file",
"purpose": "batch",
"status": "uploaded",
}
@respx.mock
async def test_file_content_of_an_uploaded_file_downloads_original_bytes() -> None:
respx.get(f"{API_BASE}/v1/files/file_07/content").respond(200, content=b'{"custom_id":"r1"}\n')
content: Final = await litellm.afile_content(
file_id="file_07", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE
)
assert content.content == b'{"custom_id":"r1"}\n'
@respx.mock
async def test_delete_file_maps_xai_deleted_object() -> None:
respx.delete(f"{API_BASE}/v1/files/file_07").respond(200, json={"id": "file_07", "deleted": True, "object": "file"})
deleted: Final = await litellm.afile_delete(
file_id="file_07", custom_llm_provider="xai", api_key=KEY, api_base=API_BASE
)
assert deleted.model_dump() == {"id": "file_07", "deleted": True, "object": "file"}
@respx.mock
async def test_create_file_falls_back_to_litellm_xai_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("XAI_API_KEY", raising=False)
monkeypatch.setattr(litellm, "xai_key", "configured-xai-key")
monkeypatch.setattr(litellm, "api_key", "generic-key-must-not-be-used")
route: Final = respx.post(f"{API_BASE}/v1/files").respond(200, json=_XAI_FILE)
await litellm.acreate_file(
file=("batch.jsonl", b'{"custom_id":"r1"}\n', "application/jsonl"),
purpose="batch",
custom_llm_provider="xai",
api_base=API_BASE,
)
assert route.calls.last.request.headers["authorization"] == "Bearer configured-xai-key"
@respx.mock
async def test_list_files_reads_data_array() -> None:
respx.get(f"{API_BASE}/v1/files").respond(200, json={"data": [_XAI_FILE], "pagination_token": None})
listed: Final = await litellm.afile_list(custom_llm_provider="xai", api_key=KEY, api_base=API_BASE)
files: Final = TypeAdapter(tuple[OpenAIFileObject, ...]).validate_python(listed)
assert [f.id for f in files] == ["file_07"]
@respx.mock
async def test_list_files_walks_every_page_by_pagination_token() -> None:
route: Final = respx.get(f"{API_BASE}/v1/files").mock(
side_effect=[
httpx.Response(200, json={"data": [_XAI_FILE], "pagination_token": "file_07"}),
httpx.Response(200, json={"data": [{**_XAI_FILE, "id": "file_08"}], "pagination_token": "file_08"}),
httpx.Response(200, json={"data": [], "pagination_token": "file_08"}),
]
)
listed: Final = await litellm.afile_list(custom_llm_provider="xai", api_key=KEY, api_base=API_BASE)
files: Final = TypeAdapter(tuple[OpenAIFileObject, ...]).validate_python(listed)
assert [f.id for f in files] == ["file_07", "file_08"]
assert [call.request.url.params.get("pagination_token") for call in route.calls] == [None, "file_07", "file_08"]
async def _retrieve_file(sync_mode: bool, file_id: str) -> None:
if sync_mode:
litellm.file_retrieve(file_id=file_id, custom_llm_provider="xai", api_key=KEY, api_base=API_BASE)
return
await litellm.afile_retrieve(file_id=file_id, custom_llm_provider="xai", api_key=KEY, api_base=API_BASE)
@pytest.mark.parametrize("sync_mode", [True, False])
@respx.mock
async def test_retrieve_file_maps_xai_not_found_to_a_404_error(sync_mode: bool) -> None:
respx.get(f"{API_BASE}/v1/files/file_gone").respond(404, json={"code": "not-found", "error": "File not found"})
with pytest.raises(BaseLLMException) as raised:
await _retrieve_file(sync_mode, "file_gone")
assert raised.value.status_code == 404
assert "File not found" in str(raised.value)

10
tests/unit/llms/xai/test_xai_chat_transformation.py
View file

@ -16,7 +16,7 @@ from litellm.types.utils import (
class TestXAIReasoningTokenFolding:
"""``_fold_reasoning_tokens_into_completion`` re-aligns xAI Usage to the OpenAI invariant."""
"""``fold_reasoning_tokens_into_completion`` re-aligns xAI Usage to the OpenAI invariant."""
@staticmethod
def _make_response(
@ -45,7 +45,7 @@ class TestXAIReasoningTokenFolding:
reasoning_tokens=312,
)
XAIChatConfig._fold_reasoning_tokens_into_completion(response)
XAIChatConfig.fold_reasoning_tokens_into_completion(response)
usage = response.usage
assert usage.completion_tokens == 322
@ -59,7 +59,7 @@ class TestXAIReasoningTokenFolding:
reasoning_tokens=312,
)
XAIChatConfig._fold_reasoning_tokens_into_completion(response)
XAIChatConfig.fold_reasoning_tokens_into_completion(response)
assert response.usage.completion_tokens == 322
@ -71,7 +71,7 @@ class TestXAIReasoningTokenFolding:
reasoning_tokens=0,
)
XAIChatConfig._fold_reasoning_tokens_into_completion(response)
XAIChatConfig.fold_reasoning_tokens_into_completion(response)
assert response.usage.completion_tokens == 10
@ -84,7 +84,7 @@ class TestXAIReasoningTokenFolding:
reasoning_tokens=312,
)
XAIChatConfig._fold_reasoning_tokens_into_completion(response)
XAIChatConfig.fold_reasoning_tokens_into_completion(response)
assert response.usage.completion_tokens == 10
assert response.usage.total_tokens == 999

49
tests/unit/test_cost_calculator.py
View file

@ -4581,6 +4581,55 @@ def test_every_openai_entry_with_a_long_context_rate_and_a_batch_rate_declares_t
assert undeclared == []
@pytest.mark.parametrize("prefix", _BATCH_RATE_PREFIXES)
def test_every_xai_entry_with_a_long_context_rate_and_a_batch_rate_declares_the_batch_tier(
_local_model_cost_map: None, prefix: str
) -> None:
undeclared: Final = [
name
for name, entry in litellm.model_cost.items()
if isinstance(entry, dict)
and entry.get("litellm_provider") == "xai"
and entry.get(f"{prefix}_above_200k_tokens") is not None
and entry.get(f"{prefix}_batches") is not None
and entry.get(f"{prefix}_above_200k_tokens_batches") is None
]
assert undeclared == []
_XAI_TIERED_BATCH_MODEL: Final = "xai/grok-4.3"
def test_xai_batch_tier_discounts_the_long_context_rate_like_the_flat_batch_rate(_local_model_cost_map: None) -> None:
info: Final = litellm.get_model_info(_XAI_TIERED_BATCH_MODEL, custom_llm_provider="xai")
flat_discount: Final = info["input_cost_per_token_batches"] / info["input_cost_per_token"]
for prefix in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost"):
tier_discount = info[f"{prefix}_above_200k_tokens_batches"] / info[f"{prefix}_above_200k_tokens"]
assert tier_discount == pytest.approx(flat_discount)
assert info[f"{prefix}_above_200k_tokens_batches"] < info[f"{prefix}_above_200k_tokens"]
@pytest.mark.parametrize(
("prompt_tokens", "tier"), [(200_000, "_above_200k_tokens_batches"), (199_999, "_batches")]
)
def test_xai_batch_cost_calculator_bills_the_200k_batch_tier_inclusively(
_local_model_cost_map: None, prompt_tokens: int, tier: str
) -> None:
from litellm.cost_calculator import batch_cost_calculator
info: Final = litellm.get_model_info(_XAI_TIERED_BATCH_MODEL, custom_llm_provider="xai")
usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=64, total_tokens=prompt_tokens + 64)
prompt_cost, completion_cost_value = batch_cost_calculator(
usage=usage, model=_XAI_TIERED_BATCH_MODEL, custom_llm_provider="xai"
)
assert prompt_cost == pytest.approx(prompt_tokens * info[f"input_cost_per_token{tier}"])
assert completion_cost_value == pytest.approx(64 * info[f"output_cost_per_token{tier}"])
def test_batch_cost_calculator_ignores_malformed_batch_tier_keys():
from litellm.cost_calculator import batch_cost_calculator

3
tests/unit/test_utils.py
View file

@ -774,6 +774,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"cache_read_input_token_cost_above_32k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_128k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_200k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_200k_tokens_batches": {"type": "number"},
"cache_read_input_token_cost_above_256k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_272k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_272k_tokens_flex": {"type": "number"},
@ -797,6 +798,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"input_cost_per_video_token": {"type": "number"},
"input_cost_per_token_above_32k_tokens": {"type": "number"},
"input_cost_per_token_above_200k_tokens": {"type": "number"},
"input_cost_per_token_above_200k_tokens_batches": {"type": "number"},
"input_cost_per_token_above_256k_tokens": {"type": "number"},
"input_cost_per_token_above_272k_tokens": {"type": "number"},
"input_cost_per_token_above_512k_tokens": {"type": "number"},
@ -897,6 +899,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"output_cost_per_token_above_32k_tokens": {"type": "number"},
"output_cost_per_token_above_128k_tokens": {"type": "number"},
"output_cost_per_token_above_200k_tokens": {"type": "number"},
"output_cost_per_token_above_200k_tokens_batches": {"type": "number"},
"output_cost_per_token_above_256k_tokens": {"type": "number"},
"output_cost_per_token_above_272k_tokens": {"type": "number"},
"output_cost_per_token_above_512k_tokens": {"type": "number"},

12
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -32981,6 +32981,8 @@ export interface components {
cache_read_input_token_cost?: number | null;
/** Cache Read Input Token Cost Above 200K Tokens */
cache_read_input_token_cost_above_200k_tokens?: number | null;
/** Cache Read Input Token Cost Above 200K Tokens Batches */
cache_read_input_token_cost_above_200k_tokens_batches?: number | null;
/** Cache Read Input Token Cost Above 200K Tokens Priority */
cache_read_input_token_cost_above_200k_tokens_priority?: number | null;
/** Cache Read Input Token Cost Above 272K Tokens */
@ -33059,6 +33061,8 @@ export interface components {
input_cost_per_token_above_128k_tokens?: number | null;
/** Input Cost Per Token Above 200K Tokens */
input_cost_per_token_above_200k_tokens?: number | null;
/** Input Cost Per Token Above 200K Tokens Batches */
input_cost_per_token_above_200k_tokens_batches?: number | null;
/** Input Cost Per Token Above 200K Tokens Priority */
input_cost_per_token_above_200k_tokens_priority?: number | null;
/** Input Cost Per Token Above 272K Tokens */
@ -33182,6 +33186,8 @@ export interface components {
output_cost_per_token_above_128k_tokens?: number | null;
/** Output Cost Per Token Above 200K Tokens */
output_cost_per_token_above_200k_tokens?: number | null;
/** Output Cost Per Token Above 200K Tokens Batches */
output_cost_per_token_above_200k_tokens_batches?: number | null;
/** Output Cost Per Token Above 200K Tokens Priority */
output_cost_per_token_above_200k_tokens_priority?: number | null;
/** Output Cost Per Token Above 272K Tokens */
@ -46768,6 +46774,8 @@ export interface components {
cache_read_input_token_cost?: number | null;
/** Cache Read Input Token Cost Above 200K Tokens */
cache_read_input_token_cost_above_200k_tokens?: number | null;
/** Cache Read Input Token Cost Above 200K Tokens Batches */
cache_read_input_token_cost_above_200k_tokens_batches?: number | null;
/** Cache Read Input Token Cost Above 200K Tokens Priority */
cache_read_input_token_cost_above_200k_tokens_priority?: number | null;
/** Cache Read Input Token Cost Above 272K Tokens */
@ -46846,6 +46854,8 @@ export interface components {
input_cost_per_token_above_128k_tokens?: number | null;
/** Input Cost Per Token Above 200K Tokens */
input_cost_per_token_above_200k_tokens?: number | null;
/** Input Cost Per Token Above 200K Tokens Batches */
input_cost_per_token_above_200k_tokens_batches?: number | null;
/** Input Cost Per Token Above 200K Tokens Priority */
input_cost_per_token_above_200k_tokens_priority?: number | null;
/** Input Cost Per Token Above 272K Tokens */
@ -46969,6 +46979,8 @@ export interface components {
output_cost_per_token_above_128k_tokens?: number | null;
/** Output Cost Per Token Above 200K Tokens */
output_cost_per_token_above_200k_tokens?: number | null;
/** Output Cost Per Token Above 200K Tokens Batches */
output_cost_per_token_above_200k_tokens_batches?: number | null;
/** Output Cost Per Token Above 200K Tokens Priority */
output_cost_per_token_above_200k_tokens_priority?: number | null;
/** Output Cost Per Token Above 272K Tokens */