scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

Merge remote-tracking branch 'origin/main' into litellm_unknown_model_spend_logs_outside_router

Browse source 
# Conflicts:
#	litellm/proxy/openai_files_endpoints/common_utils.py
This commit is contained in:
mateo-berri 2026-09-19 04:43:19 -07:00
commit aadadc647d
73 changed files with 3212 additions and 404 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
.github/workflows/_test-unit-base.yml vendored
View file

@ -37,6 +37,18 @@ on:
required: false
type: number
default: 60
test-timeout-seconds:
description: >-
Per-test ceiling enforced by pytest-timeout, covering fixture setup and
teardown as well as the test body. A test that hangs fails with a
traceback of where it was stuck instead of idling the shard until
`timeout-minutes` cancels it. Timed-out tests are excluded from reruns
because pytest-timeout arms its timer once per test and
pytest-rerunfailures reruns inside that same window, so a rerun of a
timed-out test would run with no timer at all.
required: false
type: number
default: 120
max-failures:
description: "Stop after this many failures"
required: false
@ -137,6 +149,7 @@ jobs:
MAX_FAILURES: ${{ inputs.max-failures }}
WORKERS: ${{ inputs.workers }}
RERUNS: ${{ inputs.reruns }}
TEST_TIMEOUT_SECONDS: ${{ inputs.test-timeout-seconds }}
DIST: ${{ inputs.dist }}
COVERAGE_CORE: sysmon
run: |
@ -146,6 +159,8 @@ jobs:
--maxfail="${MAX_FAILURES}" \
--reruns "${RERUNS}" \
--reruns-delay 1 \
--timeout="${TEST_TIMEOUT_SECONDS}" \
--rerun-except "from pytest-timeout" \
--durations=20 \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
@ -157,6 +172,8 @@ jobs:
-n "${WORKERS}" \
--reruns "${RERUNS}" \
--reruns-delay 1 \
--timeout="${TEST_TIMEOUT_SECONDS}" \
--rerun-except "from pytest-timeout" \
--dist="${DIST}" \
--durations=20 \
--cov=./litellm --cov=./enterprise/litellm_enterprise \

4
litellm/__init__.py
View file

@ -701,6 +701,7 @@ github_copilot_models: Set = set()
chatgpt_models: Set = set()
minimax_models: Set = set()
aws_polly_models: Set = set()
transcribe_models: Set = set()
gigachat_models: Set = set()
llamagate_models: Set = set()
reducto_models: Set = set()
@ -980,6 +981,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None:
minimax_models.add(key)
elif value.get("litellm_provider") == "aws_polly":
aws_polly_models.add(key)
elif value.get("litellm_provider") == "transcribe":
transcribe_models.add(key)
elif value.get("litellm_provider") == "gigachat":
gigachat_models.add(key)
elif value.get("litellm_provider") == "llamagate":
@ -1227,6 +1230,7 @@ def _build_models_by_provider() -> dict:
"chatgpt": chatgpt_models,
"minimax": minimax_models,
"aws_polly": aws_polly_models,
"transcribe": transcribe_models,
"gigachat": gigachat_models,
"llamagate": llamagate_models,
"reducto": reducto_models,

41
litellm/batches/batch_utils.py
View file

@ -9,6 +9,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
from litellm.llms.bedrock.batches.transformation import titan_embedding_usage_from_batch_output
from litellm.llms.vertex_ai.batches.transformation import vertex_prompt_tokens_details
from litellm.types.llms.openai import Batch
@ -52,7 +53,7 @@ def batch_cost_is_final(batch: Batch) -> bool:
async def calculate_batch_cost_and_usage(
file_content_dictionary: list[dict],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
model_name: str | None = None,
model_info: ModelInfo | None = None,
) -> BatchCostUsageResult:
@ -82,7 +83,7 @@ async def calculate_batch_cost_and_usage(
async def _handle_completed_batch(
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
model_name: str | None = None,
litellm_params: dict | None = None,
model_info: ModelInfo | None = None,
@ -168,7 +169,7 @@ class _BatchOutputLineStats:
def _classify_output_line_stats(
entries: Iterable[dict],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None,
model_info: ModelInfo | None,
) -> Iterator[_BatchOutputLineStats | _LineOutcome]:
@ -187,7 +188,7 @@ def _classify_output_line_stats(
def _safe_output_line_stats(
entry: Mapping[str, object],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None,
model_info: ModelInfo | None,
) -> _BatchOutputLineStats | None:
@ -209,7 +210,7 @@ def _safe_output_line_stats(
def _compute_output_line_stats(
entry: Mapping[str, object],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None,
model_info: ModelInfo | None,
) -> _BatchOutputLineStats:
@ -220,6 +221,7 @@ def _compute_output_line_stats(
response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
completion_details: Final = usage.completion_tokens_details
line_prompt_cost, line_completion_cost = _output_line_cost(
response_body=response_body,
usage=usage,
custom_llm_provider=custom_llm_provider,
model_name=model_name,
@ -239,19 +241,36 @@ def _compute_output_line_stats(
)
def _ocr_usage_info_from_response_body(response_body: Mapping[str, object]) -> OCRUsageInfo | None:
"""OCR results report ``usage_info`` (pages) instead of ``usage`` (tokens); None for non-OCR lines."""
raw_usage_info: Final = response_body.get("usage_info")
if not isinstance(raw_usage_info, Mapping):
return None
return OCRUsageInfo.model_validate(raw_usage_info)
def _output_line_cost(
response_body: Mapping[str, object],
usage: Usage,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None,
response_model: str | None,
model_info: ModelInfo | None,
) -> tuple[float, float]:
"""(prompt_cost, completion_cost) for one output line, priced at batch rates."""
from litellm.cost_calculator import batch_cost_calculator
from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost
cost_model: Final = (
model_name if custom_llm_provider == "bedrock" and model_name else response_model or model_name or ""
)
ocr_usage: Final = _ocr_usage_info_from_response_body(response_body)
if ocr_usage is not None:
return ocr_batch_cost(
model=cost_model,
custom_llm_provider=custom_llm_provider,
usage_info=ocr_usage,
model_info=model_info,
)
return batch_cost_calculator(
usage=usage,
model=cost_model,
@ -262,7 +281,7 @@ def _output_line_cost(
def _aggregate_batch_cost_usage_models(
entries: Iterable[dict],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None = None,
model_info: ModelInfo | None = None,
) -> BatchCostUsageResult:
@ -430,7 +449,7 @@ def _provider_output_file_id(output_file_id: str) -> str:
async def _fetch_batch_managed_file_content(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"] = "openai",
litellm_params: dict | None = None,
) -> bytes:
"""
@ -460,7 +479,7 @@ async def _fetch_batch_managed_file_content(
async def _fetch_batch_output_file_content(
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"] = "openai",
litellm_params: dict | None = None,
) -> bytes:
"""
@ -482,7 +501,7 @@ async def _fetch_batch_output_file_content(
async def count_error_file_failed_requests(
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
litellm_params: dict | None,
) -> int:
"""Count failed requests reported only in the batch's separate error file.

20
litellm/batches/main.py
View file

@ -105,9 +105,11 @@ def _resolve_timeout(
@client
async def acreate_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"],
input_file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy"] = "openai",
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
extra_body: dict[str, str] | None = None,
@ -155,9 +157,11 @@ async def acreate_batch(
@client
def create_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"],
input_file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy"] = "openai",
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
extra_body: dict[str, str] | None = None,
@ -341,7 +345,7 @@ def create_batch(
async def aretrieve_batch(
batch_id: str,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
@ -389,7 +393,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
_retrieve_batch_request: RetrieveBatchRequest,
_is_async: bool,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
] = "openai",
logging_obj: LiteLLMLoggingObj | None = None,
):
@ -497,7 +501,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
message=(
f"LiteLLM doesn't support custom_llm_provider={custom_llm_provider} for 'retrieve_batch' without a `model` kwarg. "
"Supported via this path: 'openai', 'azure', 'vertex_ai', 'anthropic'. "
"'bedrock' is supported but requires `model` to be passed so the provider config can be loaded."
"'bedrock' and 'mistral' are supported but require `model` to be passed so the provider config can be loaded."
),
model="n/a",
llm_provider=custom_llm_provider,
@ -514,7 +518,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
def retrieve_batch(
batch_id: str,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,

82
litellm/cost_calculator.py
View file

@ -141,6 +141,7 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LitellmLoggingObject,
)
from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
else:
LitellmLoggingObject = Any
@ -2114,6 +2115,87 @@ def ocr_cost(
return ocr_pages_cost + annotation_pages_cost, 0.0
_OCR_BATCH_PAGE_RATE_KEYS: Final = ("ocr_cost_per_page_batches", "ocr_cost_per_page")
_OCR_BATCH_ANNOTATION_RATE_KEYS: Final = ("annotation_cost_per_page_batches", "annotation_cost_per_page")
def ocr_batch_cost(
model: str,
custom_llm_provider: str | None,
usage_info: "OCRUsageInfo",
model_info: ModelInfo | None = None,
) -> tuple[float, float]:
"""Per-page cost of one OCR result inside a batch output file.
Batch OCR is billed per page at the ``*_batches`` rate, falling back to the
synchronous per-page rate when a model has no batch price recorded, the same
fallback ``batch_cost_calculator`` applies to per-token batch pricing. Each
per-page family (OCR pages, annotation pages) belongs to the deployment's
``model_info`` when it prices that family at either rate and to the published
cost map otherwise, so a deployment overriding one family keeps the model's
published rate for the other, and the cost map is only consulted for a family
the deployment leaves out. Returns ``(prompt_cost, completion_cost)`` with the
whole cost in the first slot, like ``ocr_cost``.
"""
pages_processed: Final = usage_info.pages_processed or 0
annotation_pages: Final = usage_info.pages_processed_annotation or 0
deployment_page_rate: Final = _first_price(model_info, *_OCR_BATCH_PAGE_RATE_KEYS)
deployment_annotation_rate: Final = _first_price(model_info, *_OCR_BATCH_ANNOTATION_RATE_KEYS)
needs_published_pricing: Final = (pages_processed > 0 and deployment_page_rate is None) or (
annotation_pages > 0 and deployment_annotation_rate is None
)
published: Final = (
_lookup_model_info_or_none(model=model, custom_llm_provider=custom_llm_provider)
if needs_published_pricing
else None
)
if needs_published_pricing and published is None:
verbose_logger.warning(
"OCR batch cost: model=%s custom_llm_provider=%s has no pricing entry; "
"billing only the per-page families the deployment prices.",
_single_log_line(model),
_single_log_line(custom_llm_provider),
)
page_rate: Final = (
deployment_page_rate
if deployment_page_rate is not None
else _first_price(published, *_OCR_BATCH_PAGE_RATE_KEYS)
)
annotation_rate: Final = (
deployment_annotation_rate
if deployment_annotation_rate is not None
else _first_price(published, *_OCR_BATCH_ANNOTATION_RATE_KEYS)
)
if page_rate is None and pages_processed > 0:
verbose_logger.warning(
"OCR batch cost: model=%s custom_llm_provider=%s reported pages_processed=%s but no "
"ocr_cost_per_page is configured; returning 0.0 cost for those pages.",
_single_log_line(model),
_single_log_line(custom_llm_provider),
pages_processed,
)
effective_annotation_rate: Final = annotation_rate if annotation_rate is not None else page_rate
return (page_rate or 0.0) * pages_processed + (effective_annotation_rate or 0.0) * annotation_pages, 0.0
def _single_log_line(value: str | None) -> str:
return str(value).replace("\n", "").replace("\r", "")
def _lookup_model_info_or_none(model: str, custom_llm_provider: str | None) -> ModelInfo | None:
try:
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; caller logs and bills 0.0
return None
def _first_price(model_info: ModelInfo | None, *keys: str) -> float | None:
if model_info is None:
return None
return next((price for price in (model_info.get(k) for k in keys) if isinstance(price, (int, float))), None)
def vector_store_search_cost(
model: str | None,
custom_llm_provider: str,

7
litellm/files/main.py
View file

@ -27,12 +27,13 @@ FileCreateProvider = Literal[
"litellm_proxy",
"manus",
"anthropic",
"mistral",
]
FileRetrieveProvider = Literal[
"openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic"
"openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic", "mistral"
]
FileDeleteProvider = Literal["openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic"]
FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic"]
FileDeleteProvider = Literal["openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic", "mistral"]
FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral"]
import litellm
from litellm import get_secret_str
from litellm.files.streaming import FileContentStreamingResponse

2
litellm/files/types.py
View file

@ -2,7 +2,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
from typing import Literal, NamedTuple
FileContentProvider = Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "manus"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "manus", "mistral"
]

4
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -647,10 +647,10 @@ class SlackAlerting(CustomBatchLogger):
event_message += f"Budget Crossed\n Total Budget:`{user_info.max_budget}`"
elif percent_left <= SLACK_ALERTING_THRESHOLD_5_PERCENT:
event = "threshold_crossed"
event_message += "5% Threshold Crossed "
event_message += "5% or less of budget remaining"
elif percent_left <= SLACK_ALERTING_THRESHOLD_15_PERCENT:
event = "threshold_crossed"
event_message += "15% Threshold Crossed"
event_message += "15% or less of budget remaining"
return event, event_message

9
litellm/integrations/otel/logger.py
View file

@ -5,7 +5,7 @@ from collections.abc import Callable, Iterator, Mapping, Sequence
from contextlib import contextmanager
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Final, cast
from opentelemetry.context import Context, attach, get_current
from opentelemetry.sdk._logs import LoggerProvider
@ -21,6 +21,7 @@ from opentelemetry.trace import (
use_span,
)
from opentelemetry.trace import TracerProvider as ApiTracerProvider
from typing_extensions import TypedDict, Unpack
import litellm
from litellm._logging import verbose_logger
@ -140,6 +141,10 @@ def _request_trace_links(context: Context | None) -> tuple[Link, ...] | None:
return (Link(anchor),) if anchor.is_valid else None
class _CustomLoggerOptions(TypedDict, total=False, extra_items=object):
pass
class _LLMCallSpan:
"""The state carried from the ``pre_call`` boundary to span close.
@ -179,7 +184,7 @@ class OpenTelemetryV2(CustomLogger):
tracer_provider: TracerProvider | None = None,
logger_provider: LoggerProvider | None = None,
meter_provider: "MeterProvider | None" = None,
**kwargs: Any,
**kwargs: Unpack[_CustomLoggerOptions],
) -> None:
super().__init__(**kwargs)
self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs)

12
litellm/litellm_core_utils/litellm_logging.py
View file

@ -371,6 +371,10 @@ _DEPLOYMENT_PRICING_KEYS: Final = (
"output_cost_per_token",
"input_cost_per_token_batches",
"output_cost_per_token_batches",
"ocr_cost_per_page",
"ocr_cost_per_page_batches",
"annotation_cost_per_page",
"annotation_cost_per_page_batches",
)
@ -386,7 +390,9 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str |
the model's published rates instead of billing as zero. Ownership is per
token direction: declaring either rate for a direction takes that whole
direction, so a published batch rate can never displace a standard rate
the deployment configured itself.
the deployment configured itself. OCR per-page rates count as declared
pricing too; they pass through as registered and ``ocr_batch_cost`` layers
the published rate under each per-page family the deployment leaves out.
"""
if model_id is None:
return None
@ -1239,8 +1245,8 @@ class Logging(LiteLLMLoggingBaseClass):
return {"error": f"Unable to parse raw request body. Got - {data}"}
return data
def _get_masked_api_base(self, api_base: str) -> str:
return str(mask_api_base_credentials(api_base))
def _get_masked_api_base(self, api_base: str | None) -> str:
return str(mask_api_base_credentials(api_base or ""))
def _pre_call(self, input, api_key, model=None, additional_args={}):
"""

54
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -1,5 +1,6 @@
from collections.abc import Mapping, Sequence
from collections.abc import Set as AbstractSet
from dataclasses import dataclass, field
from typing import Any, Final
from pydantic import BaseModel
@ -176,26 +177,49 @@ def mask_credentials_in_payload(data: object) -> object:
config-dump semantics (``None`` -> ``"None"``, tuples stringified,
objects flattened via ``__dict__``) would silently distort the record.
A container referenced from several places in ``data`` is rebuilt once and
referenced from the same places in the copy, so a shared subtree never
fans out into independent copies, and a reference back into a container
still being rebuilt (a cycle) becomes ``REDACTED``. A container nested past
``DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER`` is replaced by
``REDACTED`` rather than returned unmasked.
Sensitive-key detection is delegated to the shared
:class:`SensitiveDataMasker` so pattern updates stay in one place.
"""
return _walk_payload(data, key_is_sensitive=False, depth=0)
return _PayloadWalker().walk(data, key_is_sensitive=False, depth=0)
def _walk_payload(node: object, key_is_sensitive: bool, depth: int) -> object:
if depth >= DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER:
return node
if isinstance(node, Mapping):
return {k: _walk_payload(v, _default_masker.is_sensitive_key(k), depth + 1) for k, v in node.items()}
if isinstance(node, list):
return [_walk_payload(item, key_is_sensitive, depth + 1) for item in node]
if isinstance(node, tuple):
return tuple(_walk_payload(item, key_is_sensitive, depth + 1) for item in node)
if isinstance(node, BaseModel):
return _walk_payload(node.model_dump(), key_is_sensitive, depth)
if key_is_sensitive and isinstance(node, str) and node:
return _default_masker._mask_value(node)
return node
@dataclass(frozen=True, slots=True)
class _PayloadWalker:
_memo: dict[tuple[int, bool], tuple[object, object]] = field( # mutable-ok: memo of one walk, pins each keyed node
default_factory=dict
)
def walk(self, node: object, key_is_sensitive: bool, depth: int) -> object:
if not isinstance(node, (Mapping, list, tuple, BaseModel)):
return _default_masker._mask_value(node) if key_is_sensitive and isinstance(node, str) and node else node
if depth >= DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER:
return REDACTED
memo_key: Final = (id(node), key_is_sensitive and not isinstance(node, Mapping))
cached: Final = self._memo.get(memo_key)
if cached is not None:
return cached[1]
self._memo[memo_key] = (node, REDACTED)
rebuilt: Final = self._rebuild(node, key_is_sensitive, depth)
self._memo[memo_key] = (node, rebuilt)
return rebuilt
def _rebuild(
self, node: Mapping[str, object] | Sequence[object] | BaseModel, key_is_sensitive: bool, depth: int
) -> object:
if isinstance(node, BaseModel):
return self.walk(node.model_dump(), key_is_sensitive, depth)
if isinstance(node, Mapping):
return {k: self.walk(v, _default_masker.is_sensitive_key(k), depth + 1) for k, v in node.items()}
if isinstance(node, tuple):
return tuple(self.walk(item, key_is_sensitive, depth + 1) for item in node)
return [self.walk(item, key_is_sensitive, depth + 1) for item in node]
def mask_sensitive_keys(data: dict[str, Any], sensitive_fields: set[str]) -> dict[str, Any]:

5
litellm/llms/bedrock/chat/agentcore/transformation.py
View file

@ -6,7 +6,7 @@ https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgen
import json
from collections.abc import AsyncGenerator
from typing import TYPE_CHECKING, Any, Final, Optional, Union, cast
from typing import TYPE_CHECKING, Any, Final, Optional, Union
from urllib.parse import quote
import httpx
@ -31,6 +31,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import (
Choices,
Delta,
LlmProviders,
Message,
ModelResponse,
ModelResponseStream,
@ -872,7 +873,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
)
if client is None or not isinstance(client, AsyncHTTPHandler):
client = get_async_httpx_client(llm_provider=cast(Any, "bedrock"), params={})
client = get_async_httpx_client(llm_provider=LlmProviders.BEDROCK, params={})
verbose_logger.debug("Making async streaming request to: %s", api_base)

26
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -293,6 +293,26 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool:
return False
def _mask_presigned_request_headers(transformed_request: bytes | str | dict) -> bytes | str | dict:
"""A pre-signed request carries its auth inside its own ``headers`` key, which
logging treats as request body (only the top-level headers channel gets masked),
so mask it here before the request is handed to ``pre_call``."""
if not isinstance(transformed_request, dict):
return transformed_request
request_headers: Final = transformed_request.get("headers")
if not isinstance(request_headers, dict):
return transformed_request
from litellm.litellm_core_utils.litellm_logging import (
_get_masked_values, # pyright: ignore[reportPrivateUsage] # the shared header-masking helper has no public name
)
return { # mutable-ok: logging's curl and raw-request builders take dict
**transformed_request,
"headers": _get_masked_values(request_headers),
}
def _aws_signing_overrides(optional_params: Mapping[str, Any], litellm_params: Mapping[str, Any]) -> Mapping[str, Any]:
return MappingProxyType(
{
@ -3734,7 +3754,7 @@ class BaseLLMHTTPHandler:
"complete_input_dict": (
"<streaming media upload>"
if isinstance(transformed_request, dict) and "streaming_media_upload" in transformed_request
else transformed_request
else _mask_presigned_request_headers(transformed_request)
),
"api_base": api_base,
"headers": headers,
@ -4157,7 +4177,7 @@ class BaseLLMHTTPHandler:
input="",
api_key="",
additional_args={
"complete_input_dict": transformed_request,
"complete_input_dict": _mask_presigned_request_headers(transformed_request),
"api_base": api_base,
"headers": headers,
},
@ -4236,7 +4256,7 @@ class BaseLLMHTTPHandler:
input="",
api_key="",
additional_args={
"complete_input_dict": transformed_request,
"complete_input_dict": _mask_presigned_request_headers(transformed_request),
"api_base": api_base,
"headers": headers,
"batch_id": batch_id,

0
litellm/llms/mistral/batches/__init__.py Normal file
View file

220
litellm/llms/mistral/batches/transformation.py Normal file
View file

@ -0,0 +1,220 @@
"""
Mistral Batch API. Reference: https://docs.mistral.ai/api/#tag/batch
Mistral runs one model per job (set on the job, not per input line) and accepts
``/v1/ocr`` as a batch endpoint, which is how OCR gets its 50% batch discount.
Output and error files are OpenAI-shaped JSONL (``{custom_id, response: {status_code, body}}``),
so the shared batch cost accounting reads them without a provider branch.
"""
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
import httpx
from openai.types.batch import BatchRequestCounts
from openai.types.batch import Errors as BatchErrors
from openai.types.batch_error import BatchError
from pydantic import BaseModel, ConfigDict
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
from litellm.types.utils import LiteLLMBatch, LlmProviders
from ..common_utils import get_mistral_api_base, get_mistral_auth_headers, mistral_error
MistralBatchStatus: TypeAlias = Literal[
"QUEUED", "RUNNING", "SUCCESS", "FAILED", "TIMEOUT_EXCEEDED", "CANCELLATION_REQUESTED", "CANCELLED"
]
OpenAIBatchStatus: TypeAlias = Literal[
"validating", "failed", "in_progress", "finalizing", "completed", "expired", "cancelling", "cancelled"
]
_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) # mutable-ok: frozen at module scope
_STATUS_MAP: Final[MappingProxyType[MistralBatchStatus, OpenAIBatchStatus]] = MappingProxyType(
{
"QUEUED": "validating",
"RUNNING": "in_progress",
"SUCCESS": "completed",
"FAILED": "failed",
"TIMEOUT_EXCEEDED": "expired",
"CANCELLATION_REQUESTED": "cancelling",
"CANCELLED": "cancelled",
}
)
class MistralCreateBatchJobRequest(TypedDict):
"""Body of ``POST /v1/batch/jobs``."""
input_files: ReadOnly[tuple[str, ...]]
endpoint: ReadOnly[str]
model: ReadOnly[str]
metadata: NotRequired[ReadOnly[Mapping[str, str]]]
class MistralPresignedRequest(TypedDict):
"""A fully-formed request the shared HTTP handler sends as-is (its ``method`` branch)."""
method: ReadOnly[Literal["GET"]]
url: ReadOnly[str]
headers: ReadOnly[Mapping[str, str]]
class MistralBatchError(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
message: str
count: int = 1
class MistralBatchJob(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
id: str
input_files: tuple[str, ...] = ()
endpoint: str
model: str | None = None
status: MistralBatchStatus
created_at: int
started_at: int | None = None
completed_at: int | None = None
total_requests: int = 0
completed_requests: int = 0
succeeded_requests: int = 0
failed_requests: int = 0
output_file: str | None = None
error_file: str | None = None
errors: tuple[MistralBatchError, ...] = ()
metadata: dict[str, str] | None = None # mutable-ok: LiteLLMBatch.metadata is typed as dict
def _to_batch_errors(errors: Sequence[MistralBatchError]) -> BatchErrors | None:
if not errors:
return None
return BatchErrors(
object="list",
data=[ # mutable-ok: openai Batch.Errors.data is typed as list
BatchError(message=f"{e.message} (x{e.count})" if e.count > 1 else e.message) for e in errors
],
)
def _to_litellm_batch(job: MistralBatchJob) -> LiteLLMBatch:
status: Final = _STATUS_MAP[job.status]
terminal_at: Final = job.completed_at
return LiteLLMBatch(
id=job.id,
object="batch",
endpoint=job.endpoint,
input_file_id=job.input_files[0] if job.input_files else "",
completion_window="24h",
status=status,
created_at=job.created_at,
in_progress_at=job.started_at,
completed_at=terminal_at if status == "completed" else None,
failed_at=terminal_at if status == "failed" else None,
expired_at=terminal_at if status == "expired" else None,
cancelled_at=terminal_at if status == "cancelled" else None,
output_file_id=job.output_file,
error_file_id=job.error_file,
errors=_to_batch_errors(job.errors),
request_counts=BatchRequestCounts(
total=job.total_requests,
completed=job.succeeded_requests,
failed=job.failed_requests,
),
metadata=job.metadata,
)
class MistralBatchesConfig(BaseBatchesConfig):
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.MISTRAL
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: BaseBatchesConfig signature
return get_mistral_auth_headers(headers, api_key)
def get_complete_batch_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
data: CreateBatchRequest,
) -> str:
return f"{get_mistral_api_base(api_base)}/v1/batch/jobs"
def transform_create_batch_request(
self,
model: str,
create_batch_data: CreateBatchRequest,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> dict[str, object]: # mutable-ok: BaseBatchesConfig signature
input_file_id: Final = create_batch_data.get("input_file_id")
endpoint: Final = create_batch_data.get("endpoint")
if input_file_id is None or endpoint is None:
raise ValueError("input_file_id and endpoint are required to create a Mistral batch job")
metadata: Final = create_batch_data.get("metadata")
body: Final = (
MistralCreateBatchJobRequest(
input_files=(input_file_id,), endpoint=endpoint, model=model, metadata=metadata
)
if metadata
else MistralCreateBatchJobRequest(input_files=(input_file_id,), endpoint=endpoint, model=model)
)
return dict(body) # mutable-ok: BaseBatchesConfig signature
def transform_create_batch_response(
self,
model: str | None,
raw_response: httpx.Response,
logging_obj: object,
litellm_params: Mapping[str, object],
) -> LiteLLMBatch:
return _to_litellm_batch(MistralBatchJob.model_validate(raw_response.json()))
def transform_retrieve_batch_request(
self,
batch_id: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> dict[str, object]: # mutable-ok: BaseBatchesConfig signature
encoded_batch_id: Final = encode_url_path_segment(batch_id, field_name="batch_id")
api_base: Final = litellm_params.get("api_base")
api_key: Final = litellm_params.get("api_key")
request: Final = MistralPresignedRequest(
method="GET",
url=f"{get_mistral_api_base(api_base if isinstance(api_base, str) else None)}/v1/batch/jobs/{encoded_batch_id}",
headers=get_mistral_auth_headers(_NO_HEADERS, api_key if isinstance(api_key, str) else None),
)
return dict(request) # mutable-ok: BaseBatchesConfig signature
def transform_retrieve_batch_response(
self,
model: str | None,
raw_response: httpx.Response,
logging_obj: object,
litellm_params: Mapping[str, object],
) -> LiteLLMBatch:
return _to_litellm_batch(MistralBatchJob.model_validate(raw_response.json()))
def get_error_class(
self, error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers
) -> BaseLLMException:
return mistral_error(error_message, status_code, headers)

41
litellm/llms/mistral/common_utils.py Normal file
View file

@ -0,0 +1,41 @@
from collections.abc import Mapping
from typing import Final
import httpx
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
MISTRAL_API_BASE: Final = "https://api.mistral.ai"
MISTRAL_API_KEY_ENV_VAR: Final = "MISTRAL_API_KEY"
class MistralError(BaseLLMException):
pass
def get_mistral_api_base(api_base: str | None) -> str:
"""Return the Mistral origin without a trailing ``/v1``, so callers can append ``/v1/<route>``."""
resolved: Final = (api_base or get_secret_str("MISTRAL_API_BASE") or MISTRAL_API_BASE).rstrip("/")
return resolved.removesuffix("/v1")
def get_mistral_auth_headers(
headers: Mapping[str, str], api_key: str | None
) -> dict[str, str]: # mutable-ok: BaseConfig.validate_environment contract returns dict
resolved_key: Final = api_key or get_secret_str(MISTRAL_API_KEY_ENV_VAR)
if resolved_key is None:
raise ValueError(
"Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params"
)
return dict(headers, Authorization=f"Bearer {resolved_key}") # mutable-ok: BaseConfig contract returns dict
def mistral_error(error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers) -> MistralError:
return MistralError(
status_code=status_code,
message=error_message,
headers=headers
if isinstance(headers, httpx.Headers)
else httpx.Headers(dict(headers)), # mutable-ok: httpx.Headers takes a dict
)

0
litellm/llms/mistral/files/__init__.py Normal file
View file

267
litellm/llms/mistral/files/transformation.py Normal file
View file

@ -0,0 +1,267 @@
"""
Mistral Files API. Reference: https://docs.mistral.ai/api/#tag/files
Mistral's file objects already carry the OpenAI field names (id, bytes, created_at,
filename, purpose), so this config is URL routing, auth, and a purpose mapping:
Mistral only accepts ``fine-tune``, ``batch`` and ``ocr`` as upload purposes, while files
other Mistral products created read back with purposes outside that set and map onto ``user_data``.
"""
import time
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
import httpx
from openai.types.file_deleted import FileDeleted
from pydantic import BaseModel, ConfigDict
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj
from litellm.types.llms.openai import (
CreateFileRequest,
FileContentRequest,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
OpenAIFilesPurpose,
)
from litellm.types.utils import LlmProviders
from ..common_utils import get_mistral_api_base, get_mistral_auth_headers, mistral_error
MistralFilePurpose: TypeAlias = Literal["fine-tune", "batch", "ocr"]
_OPENAI_PURPOSE_BY_MISTRAL: Final[Mapping[str, OpenAIFilesPurpose]] = MappingProxyType(
{"fine-tune": "fine-tune", "batch": "batch", "ocr": "user_data"}
)
_OPENAI_PURPOSE_FOR_UNMAPPED: Final[OpenAIFilesPurpose] = "user_data"
_MISTRAL_PURPOSE_BY_OPENAI: Final[Mapping[str, MistralFilePurpose]] = MappingProxyType(
{"fine-tune": "fine-tune", "batch": "batch", "ocr": "ocr", "user_data": "ocr"}
)
_SUPPORTED_PURPOSES: Final = ", ".join(_MISTRAL_PURPOSE_BY_OPENAI)
_NO_QUERY_PARAMS: Final[dict[str, str]] = {} # mutable-ok: BaseFilesConfig request transforms return tuple[str, dict]
class MistralMultipartUpload(TypedDict):
"""``files=`` payload for ``POST /v1/files``: each value is an httpx multipart tuple."""
file: ReadOnly[tuple[str, object, str]]
purpose: ReadOnly[tuple[None, MistralFilePurpose]]
class MistralFile(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
id: str
bytes: int = 0
created_at: int | None = None
filename: str = ""
purpose: str = "batch"
expires_at: int | None = None
class MistralFileList(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
data: tuple[MistralFile, ...] = ()
class MistralFileDeleted(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
id: str
deleted: bool = True
def _to_openai_file_object(file: MistralFile) -> OpenAIFileObject:
return OpenAIFileObject(
id=file.id,
bytes=file.bytes,
created_at=file.created_at if file.created_at is not None else int(time.time()),
filename=file.filename,
object="file",
purpose=_to_openai_purpose(file.purpose),
status="uploaded",
expires_at=file.expires_at,
)
def _to_openai_purpose(purpose: str) -> OpenAIFilesPurpose:
return _OPENAI_PURPOSE_BY_MISTRAL.get(purpose, _OPENAI_PURPOSE_FOR_UNMAPPED)
def _to_mistral_purpose(purpose: str) -> MistralFilePurpose:
"""``user_data`` is what an OCR file reads back as, since OpenAI's purpose literal has no ``ocr``,
so it maps back onto ``ocr``. Every other purpose Mistral lacks is rejected: silently rewriting
it to ``batch`` would let an upload skip the proxy's batch-file validation and guardrails, which
only run when the caller says ``purpose=batch``."""
mistral_purpose: Final = _MISTRAL_PURPOSE_BY_OPENAI.get(purpose)
if mistral_purpose is None:
raise mistral_error(
f"Mistral does not support purpose={purpose!r}. Use one of: {_SUPPORTED_PURPOSES}",
status_code=400,
headers=httpx.Headers(),
)
return mistral_purpose
def _api_base_from(litellm_params: Mapping[str, object]) -> str:
api_base: Final = litellm_params.get("api_base")
return get_mistral_api_base(api_base if isinstance(api_base, str) else None)
class MistralFilesConfig(BaseFilesConfig):
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.MISTRAL
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
stream: bool | None = None,
) -> str:
return f"{get_mistral_api_base(api_base)}/v1/files"
def _file_url(self, file_id: str, litellm_params: Mapping[str, object], suffix: str = "") -> str:
encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id")
return f"{_api_base_from(litellm_params)}/v1/files/{encoded_file_id}{suffix}"
def get_error_class(
self, error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers
) -> BaseLLMException:
return mistral_error(error_message, status_code, headers)
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[object],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: BaseFilesConfig signature
return get_mistral_auth_headers(headers, api_key)
def get_supported_openai_params(
self, model: str
) -> list[OpenAICreateFileRequestOptionalParams]: # mutable-ok: BaseFilesConfig signature
return ["purpose"] # mutable-ok: BaseFilesConfig signature
def map_openai_params(
self,
non_default_params: Mapping[str, object],
optional_params: dict[str, object], # mutable-ok: BaseConfig signature, returned as-is
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: BaseConfig signature
return optional_params
def transform_create_file_request(
self,
model: str,
create_file_data: CreateFileRequest,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> dict[str, object]: # mutable-ok: BaseFilesConfig signature
if "file" not in create_file_data:
raise ValueError("File data is required")
extracted: Final = extract_file_data(create_file_data["file"])
filename: Final = extracted["filename"] or f"file_{int(time.time())}.jsonl"
content_type: Final = extracted.get("content_type") or "application/octet-stream"
upload: Final = MistralMultipartUpload(
file=(filename, extracted["content"], content_type),
purpose=(None, _to_mistral_purpose(create_file_data.get("purpose") or "batch")),
)
return dict(upload) # mutable-ok: BaseFilesConfig signature
def transform_create_file_response(
self,
model: str | None,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
) -> OpenAIFileObject:
return _to_openai_file_object(MistralFile.model_validate(raw_response.json()))
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature
return self._file_url(file_id, litellm_params), _NO_QUERY_PARAMS
def transform_retrieve_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
) -> OpenAIFileObject:
return _to_openai_file_object(MistralFile.model_validate(raw_response.json()))
def transform_delete_file_request(
self,
file_id: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature
return self._file_url(file_id, litellm_params), _NO_QUERY_PARAMS
def transform_delete_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
) -> FileDeleted:
deleted: Final = MistralFileDeleted.model_validate(raw_response.json())
return FileDeleted(id=deleted.id, deleted=deleted.deleted, object="file")
def transform_list_files_request(
self,
purpose: str | None,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature
url: Final = f"{_api_base_from(litellm_params)}/v1/files"
if not purpose:
return url, _NO_QUERY_PARAMS
return url, {"purpose": _to_mistral_purpose(purpose)} # mutable-ok: BaseFilesConfig signature returns dict
def transform_list_files_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
) -> list[OpenAIFileObject]: # mutable-ok: BaseFilesConfig signature
return [ # mutable-ok: BaseFilesConfig signature
_to_openai_file_object(f) for f in MistralFileList.model_validate(raw_response.json()).data
]
def transform_file_content_request(
self,
file_content_request: FileContentRequest,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature
file_id: Final = file_content_request.get("file_id")
if file_id is None:
raise ValueError("file_id is required to download file content")
return self._file_url(file_id, litellm_params, suffix="/content"), _NO_QUERY_PARAMS
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: Mapping[str, object],
) -> HttpxBinaryResponseContent:
return HttpxBinaryResponseContent(response=raw_response)

2
litellm/main.py
View file

@ -5108,7 +5108,7 @@ def completion(
messages = validate_and_fix_openai_messages(messages=messages)
tools = validate_and_fix_openai_tools(tools=tools)
# validate tool_choice
tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice)
tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice, model=model)
# validate optional params
stop = validate_openai_optional_params(stop=stop)
thinking = validate_and_fix_thinking_param(thinking=thinking)

40
litellm/model_prices_and_context_window_backup.json
View file

@ -37476,51 +37476,66 @@
"mistral/mistral-ocr-latest": {
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.004,
"ocr_cost_per_page_batches": 0.002,
"annotation_cost_per_page": 0.005,
"annotation_cost_per_page_batches": 0.0025,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
"mistral/mistral-ocr-4-0": {
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.004,
"ocr_cost_per_page_batches": 0.002,
"annotation_cost_per_page": 0.005,
"annotation_cost_per_page_batches": 0.0025,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
"mistral/mistral-ocr-4-1": {
"annotation_cost_per_page": 0.005,
"annotation_cost_per_page_batches": 0.0025,
"litellm_provider": "mistral",
"mode": "ocr",
"ocr_cost_per_page": 0.004,
"ocr_cost_per_page_batches": 0.002,
"source": "https://docs.mistral.ai/models/model-cards/ocr-4-1",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
]
},
"mistral/mistral-ocr-2505-completion": {
"deprecation_date": "2026-05-31",
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.001,
"ocr_cost_per_page_batches": 0.0005,
"annotation_cost_per_page": 0.003,
"annotation_cost_per_page_batches": 0.0015,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
"mistral/mistral-ocr-2512": {
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.002,
"ocr_cost_per_page_batches": 0.001,
"annotation_cost_per_page": 0.003,
"annotation_cost_per_page_batches": 0.0015,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
@ -63741,31 +63756,40 @@
"mistral/mistral-ocr-3": {
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.002,
"ocr_cost_per_page_batches": 0.001,
"annotation_cost_per_page": 0.003,
"annotation_cost_per_page_batches": 0.0015,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
"mistral/mistral-ocr-3-0": {
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.002,
"ocr_cost_per_page_batches": 0.001,
"annotation_cost_per_page": 0.003,
"annotation_cost_per_page_batches": 0.0015,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
"mistral/mistral-ocr-4": {
"annotation_cost_per_page": 0.005,
"annotation_cost_per_page_batches": 0.0025,
"litellm_provider": "mistral",
"mode": "ocr",
"ocr_cost_per_page": 0.004,
"ocr_cost_per_page_batches": 0.002,
"source": "https://docs.mistral.ai/models/model-cards/ocr-4-1",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
]
},
"mistral/voxtral-mini-latest": {

2
litellm/provider_endpoints_support_backup.json
View file

@ -1462,7 +1462,7 @@
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"batches": true,
"rerank": false,
"ocr": true,
"a2a": true,

1
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -11155,7 +11155,6 @@
"type": "null"
}
],
"default": "v1",
"description": "API version for Javelin service",
"title": "Api Version"
},

39
litellm/proxy/batches_endpoints/endpoints.py
View file

@ -32,16 +32,18 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
from litellm.proxy.openai_files_endpoints.common_utils import (
BATCH_CREATE_HIDDEN_PARAM,
_is_base64_encoded_unified_file_id,
add_deployment_model_info,
add_internal_model_credentials,
apply_team_provider_credentials,
authorize_model_for_key,
batch_cost_poller_is_active,
decode_model_from_file_id,
encode_batch_response_ids,
encode_file_id_with_model,
ensure_batch_response_managed_file_ids,
get_authorized_credentials_for_model,
get_batch_from_database,
get_batch_id_from_unified_batch_id,
get_credentials_for_model,
get_model_id_from_unified_batch_id,
get_models_from_unified_file_id,
get_original_file_id,
@ -223,9 +225,10 @@ async def create_batch(
# SCENARIO 1: File ID is encoded with model info
if model_from_file_id is not None and input_file_id:
credentials = get_credentials_for_model(
credentials = await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=model_from_file_id,
user_api_key_dict=user_api_key_dict,
operation_context="batch creation (file created with model)",
)
@ -290,6 +293,7 @@ async def create_batch(
detail={"error": f"Expected 1 model, got {len(target_model_names)}"},
)
model: Final = target_model_names[0]
await authorize_model_for_key(model_id=model, llm_router=llm_router, user_api_key_dict=user_api_key_dict)
_create_batch_data["model"] = model
resolved_storage_url: Final = await _resolve_managed_input_file_storage_url(input_file_id)
@ -315,9 +319,10 @@ async def create_batch(
# SCENARIO 2 & 3: Model from header/query OR custom_llm_provider fallback
if model_param:
# SCENARIO 2: Use model-based routing from header/query/body
credentials = get_credentials_for_model(
credentials = await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=model_param,
user_api_key_dict=user_api_key_dict,
operation_context="batch creation",
)
@ -466,6 +471,17 @@ async def retrieve_batch(
route_type="aretrieve_batch",
)
unified_model_id: Final = get_model_id_from_unified_batch_id(unified_batch_id) if unified_batch_id else None
if unified_model_id is not None:
resolved_unified_model: Final = (
llm_router.resolve_model_name_from_model_id(unified_model_id) if llm_router is not None else None
)
await authorize_model_for_key(
model_id=resolved_unified_model or unified_model_id,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
)
# FIX: First, try to read from ManagedObjectTable for consistent state
managed_files_obj: Final = proxy_logging_obj.get_proxy_hook("managed_files")
from litellm.proxy.proxy_server import prisma_client
@ -546,9 +562,10 @@ async def retrieve_batch(
# Retrieve from provider (for non-terminal states or if DB lookup failed)
# SCENARIO 1: Batch ID is encoded with model info
if model_from_id is not None:
credentials: Final = get_credentials_for_model(
credentials: Final = await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=model_from_id,
user_api_key_dict=user_api_key_dict,
operation_context="batch retrieval (batch created with model)",
)
@ -564,6 +581,7 @@ async def retrieve_batch(
# so litellm.aretrieve_batch can load BedrockBatchesConfig. Without
# it the call falls into the legacy provider switch and 400s.
data["model"] = model_from_id
add_deployment_model_info(data=data, llm_router=llm_router, model_id=model_from_id)
# Retrieve batch using model credentials
response = await litellm.aretrieve_batch(
@ -588,7 +606,7 @@ async def retrieve_batch(
add_internal_model_credentials(
data=data,
llm_router=llm_router,
model_id=get_model_id_from_unified_batch_id(unified_batch_id),
model_id=unified_model_id,
)
response = await llm_router.aretrieve_batch(**data)
@ -772,9 +790,10 @@ async def list_batches(
data.get("model") or request.query_params.get("model") or request.headers.get("x-litellm-model")
):
# SCENARIO 2: Use model-based routing from header/query/body
credentials: Final = get_credentials_for_model(
credentials: Final = await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=model_param,
user_api_key_dict=user_api_key_dict,
operation_context="batch listing",
)
@ -961,9 +980,10 @@ async def cancel_batch(
# SCENARIO 1: Batch ID is encoded with model info
if model_from_id is not None:
credentials: Final = get_credentials_for_model(
credentials: Final = await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=model_from_id,
user_api_key_dict=user_api_key_dict,
operation_context="batch cancellation (batch created with model)",
)
@ -1002,6 +1022,11 @@ async def cancel_batch(
status_code=400,
detail={"error": "Invalid LiteLLM managed batch ID. Missing model_id."},
)
await authorize_model_for_key(
model_id=llm_router.resolve_model_name_from_model_id(model_id_from_batch) or model_id_from_batch,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
)
data["model"] = model_id_from_batch
data["batch_id"] = get_batch_id_from_unified_batch_id(unified_batch_id)
response = await llm_router.acancel_batch(**data)

11
litellm/proxy/guardrails/guardrail_hooks/alice/alice.py
View file

@ -11,14 +11,13 @@ from collections.abc import Mapping
from itertools import islice
from typing import (
TYPE_CHECKING,
Any, # noqa: TID251 # **kwargs forwards verbatim to CustomGuardrail.__init__; see ruff-strict.toml
Final,
Literal,
Optional,
)
import httpx
from typing_extensions import NotRequired, ReadOnly, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack
from litellm._logging import verbose_proxy_logger
from litellm.exceptions import GuardrailRaisedException, Timeout
@ -92,6 +91,10 @@ class AliceVerdict(TypedDict):
replacements: ReadOnly[NotRequired["tuple[AliceReplacement, ...]"]]
class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object):
pass
class AliceGuardrailMissingSecrets(Exception):
"""Raised when the Alice API key is not configured."""
@ -144,7 +147,9 @@ class AliceGuardrail(CustomGuardrail):
api_key: str | None = None,
api_base: str | None = None,
unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed",
**kwargs: Any, # kwargs-ok: forwarded verbatim to CustomGuardrail.__init__, whose param list is wide and evolving
**kwargs: Unpack[ # kwargs-ok: forwarded verbatim to CustomGuardrail.__init__, whose param list is wide and evolving
_CustomGuardrailOptions
],
) -> None:
self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)

14
litellm/proxy/guardrails/guardrail_hooks/azure/base.py
View file

@ -20,6 +20,15 @@ AZURE_CONTENT_SAFETY_MAX_TEXT_LENGTH: Final = 10000
# chunk of N characters consumes ceil(N / 1000) text records.
AZURE_CONTENT_SAFETY_TEXT_RECORD_LENGTH: Final = 1000
AZURE_CONTENT_SAFETY_DEFAULT_API_VERSION: Final = "2024-09-01"
JAVELIN_API_VERSION_STORED_BY_OLDER_RELEASES: Final = "v1"
def resolve_content_safety_api_version(configured: str | None) -> str:
if not configured or configured == JAVELIN_API_VERSION_STORED_BY_OLDER_RELEASES:
return AZURE_CONTENT_SAFETY_DEFAULT_API_VERSION
return configured
class AzureGuardrailBase:
"""
@ -43,7 +52,7 @@ class AzureGuardrailBase:
self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
self.api_key = api_key
self.api_base = api_base
self.api_version: str = kwargs.get("api_version") or "2024-09-01"
self.api_version: str | None = kwargs.get("api_version")
async def _post_to_content_safety(self, endpoint_path: str, request_body: dict[str, object]) -> dict[str, Any]:
"""POST to an Azure Content Safety endpoint with standard auth headers.
@ -56,7 +65,8 @@ class AzureGuardrailBase:
Returns:
Parsed JSON response dict.
"""
url: Final = f"{self.api_base}/contentsafety/{endpoint_path}?api-version={self.api_version}"
api_version: Final = resolve_content_safety_api_version(self.api_version)
url: Final = f"{self.api_base}/contentsafety/{endpoint_path}?api-version={api_version}"
headers: Final = {
"Ocp-Apim-Subscription-Key": self.api_key,
"Content-Type": "application/json",

10
litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py
View file

@ -2,10 +2,10 @@
import os
import time
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol
from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol
from fastapi import HTTPException
from typing_extensions import NotRequired, ReadOnly, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_guardrail import (
@ -38,6 +38,10 @@ class _GraySwanMonitorResponse(TypedDict):
ipi: ReadOnly[NotRequired[bool | None]]
class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object):
pass
class _GraySwanMonitorHTTPResponse(Protocol):
def raise_for_status(self) -> object: ...
@ -103,7 +107,7 @@ class GraySwanGuardrail(CustomGuardrail):
streaming_sampling_rate: int = 5,
fail_open: bool | None = True,
guardrail_timeout: float | None = 30.0,
**kwargs: Any,
**kwargs: Unpack[_CustomGuardrailOptions],
) -> None:
self.async_handler: _GraySwanMonitorHTTPClient = get_async_httpx_client(
llm_provider=httpxSpecialProvider.GuardrailCallback

18
litellm/proxy/guardrails/guardrail_hooks/promptguard/promptguard.py
View file

@ -7,9 +7,10 @@ before and after LLM calls.
"""
import os
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict
from typing import TYPE_CHECKING, Final, Literal, Optional, TypedDict
from typing_extensions import ReadOnly
from typing_extensions import ReadOnly, Unpack
from typing_extensions import TypedDict as ExtraItemsTypedDict
from litellm._logging import verbose_proxy_logger
from litellm.exceptions import GuardrailRaisedException
@ -53,6 +54,10 @@ class PromptGuardHTTPView(TypedDict):
guard_response: ReadOnly[PromptGuardGuardAPIResponse]
class _CustomGuardrailOptions(ExtraItemsTypedDict, total=False, extra_items=object):
supported_event_hooks: ReadOnly[list[GuardrailEventHooks] | None]
class PromptGuardMissingCredentials(Exception):
pass
@ -63,7 +68,7 @@ class PromptGuardGuardrail(CustomGuardrail):
api_key: str | None = None,
api_base: str | None = None,
block_on_error: bool | None = None,
**kwargs: Any,
**kwargs: Unpack[_CustomGuardrailOptions],
) -> None:
self.api_key = api_key or os.environ.get(
"PROMPTGUARD_API_KEY",
@ -92,9 +97,12 @@ class PromptGuardGuardrail(CustomGuardrail):
llm_provider=httpxSpecialProvider.GuardrailCallback,
)
kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
options: Final[_CustomGuardrailOptions] = {
"supported_event_hooks": list(self.get_supported_event_hooks()),
**kwargs,
}
super().__init__(**kwargs)
super().__init__(**options)
@staticmethod
def get_config_model() -> type["GuardrailConfigModel"] | None:

11
litellm/proxy/guardrails/guardrail_hooks/singulr/singulr.py
View file

@ -24,6 +24,7 @@ from litellm.llms.custom_httpx.http_handler import (
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.llms.openai import AllMessageValues
from litellm.types.proxy.guardrails.guardrail_hooks.base import (
GuardrailConfigModel,
)
@ -40,7 +41,7 @@ from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs
_DEFAULT_API_BASE: Final = "http://localhost:8003"
_GUARD_ENDPOINT: Final = "/api/v1/ai-gateway/litellm-v2"
_DEFAULT_TIMEOUT: Final = 30.0
_EMPTY_MAPPING: Final[Mapping[str, Any]] = MappingProxyType({})
_EMPTY_MAPPING: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({})
_MCP_MODEL_PREFIX: Final = "MCP:"
@ -159,7 +160,7 @@ class SingulrGuardrail(CustomGuardrail):
return {key: value for key, value in resolved if value} # mutable-ok: short-lived JSON payload dict
@staticmethod
def _build_user_message(text: str) -> Mapping[str, Any]:
def _build_user_message(text: str) -> Mapping[str, str]:
return {"role": "user", "content": text} # mutable-ok: short-lived JSON payload dict
def _build_headers(self) -> Mapping[str, str]:
@ -224,7 +225,7 @@ class SingulrGuardrail(CustomGuardrail):
self,
inputs: GenericGuardrailAPIInputs,
texts: Sequence[str],
structured_messages: Sequence[Any],
structured_messages: Sequence[AllMessageValues],
request_data: Mapping[str, Any],
) -> GenericGuardrailAPIInputs:
messages: Final = (
@ -271,12 +272,12 @@ class SingulrGuardrail(CustomGuardrail):
return request_data.get("mcp_tool_name") or request_data.get("name")
@staticmethod
def _mcp_arguments(request_data: Mapping[str, Any]) -> object:
def _mcp_arguments(request_data: Mapping[str, object]) -> object:
arguments: Final = request_data.get("mcp_arguments")
return arguments if arguments is not None else request_data.get("arguments")
@staticmethod
def _is_mcp_call(request_data: Mapping[str, Any], logging_obj: LiteLLMLoggingObj | None) -> bool:
def _is_mcp_call(request_data: Mapping[str, object], logging_obj: LiteLLMLoggingObj | None) -> bool:
call_type: Final = logging_obj.call_type if logging_obj is not None else request_data.get("call_type")
if call_type is not None:
return call_type == CallTypes.call_mcp_tool.value

33
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -1,6 +1,6 @@
import asyncio
import traceback
from collections.abc import Callable, Sequence
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, cast
@ -446,17 +446,26 @@ class _ProxyDBLogger(CustomLogger):
f"Cost tracking failed for model={model}.\nDebug info - {cost_tracking_failure_debug_info}\nAdd custom pricing - https://docs.litellm.ai/docs/proxy/custom_pricing"
)
except Exception as e:
error_msg = f"Error in tracking cost callback - {e}\n Traceback:{traceback.format_exc()}"
model = kwargs.get("model", "")
metadata = get_litellm_metadata_from_kwargs(kwargs=kwargs)
litellm_metadata: Final = kwargs.get("litellm_params", {}).get("litellm_metadata", {})
old_metadata: Final = kwargs.get("litellm_params", {}).get("metadata", {})
call_type = kwargs.get("call_type", "")
error_msg += f"\n Args to _PROXY_track_cost_callback\n model: {model}\n chosen_metadata: {metadata}\n litellm_metadata: {litellm_metadata}\n old_metadata: {old_metadata}\n call_type: {call_type}\n"
failing_model: Final = kwargs.get("model", "")
failing_call_type: Final = kwargs.get("call_type", "")
error_msg: Final = (
f"Error in tracking cost callback - {e}\n Traceback:{traceback.format_exc()}\n"
f" Args to _PROXY_track_cost_callback\n model: {failing_model}\n call_type: {failing_call_type}\n"
)
failing_litellm_params: Final = kwargs.get("litellm_params") or {}
verbose_proxy_logger.debug(
"Cost tracking callback failed for model=%s call_type=%s;"
" chosen_metadata keys=%s litellm_metadata keys=%s old_metadata keys=%s",
failing_model,
failing_call_type,
_metadata_keys(get_litellm_metadata_from_kwargs(kwargs=kwargs)),
_metadata_keys(failing_litellm_params.get("litellm_metadata")),
_metadata_keys(failing_litellm_params.get("metadata")),
)
asyncio.create_task(
proxy_logging_obj.failed_tracking_alert(
error_message=error_msg,
failing_model=model,
failing_model=failing_model,
)
)
@ -614,6 +623,12 @@ def _should_track_cost_callback(
return call_type in _UNATTRIBUTED_TRACKABLE_CALL_TYPES
def _metadata_keys(metadata: object) -> tuple[str, ...]:
if not isinstance(metadata, Mapping):
return ()
return tuple(sorted(str(key) for key in metadata))
def _get_budget_reservation_from_metadata(metadata: dict) -> dict | None:
metadata_budget_reservation: Final = metadata.get("user_api_key_budget_reservation")
if isinstance(metadata_budget_reservation, dict):

80
litellm/proxy/openai_files_endpoints/common_utils.py
View file

@ -350,6 +350,10 @@ def get_credentials_for_model(
"""
Retrieve API credentials for a model from the LLM Router.
Does not check whether the caller may use ``model_id``; use
``get_authorized_credentials_for_model`` for anything driven by a caller-supplied
model name (request body, header, query param, or a model-encoded resource id).
Args:
llm_router: LiteLLM Router instance
model_id: Model name or deployment ID
@ -381,6 +385,48 @@ def get_credentials_for_model(
return credentials
async def authorize_model_for_key(
model_id: str,
llm_router: Optional["Router"],
user_api_key_dict: "UserAPIKeyAuth",
) -> None:
"""
Enforce the caller's model grants on a model name the auth layer never saw.
The files and batches routes carry their model in a header, query param, or a
model-encoded resource id rather than the request body, so ``user_api_key_auth``
cannot check it. Run the same key, team (incl. team-member and access-group
fallbacks), org and project allowlist checks a chat request would get, so a
restricted key cannot borrow another deployment's server-side credentials.
Raises:
ProxyException (403): the caller is not allowed to use ``model_id``
"""
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
await can_key_call_resolved_model(
model=model_id,
llm_model_list=None,
valid_token=user_api_key_dict,
llm_router=llm_router,
)
async def get_authorized_credentials_for_model(
llm_router: Optional["Router"],
model_id: str,
user_api_key_dict: "UserAPIKeyAuth",
operation_context: str = "file operation",
) -> dict: # mutable-ok: same contract as get_credentials_for_model, callers merge it into request data
"""``get_credentials_for_model`` gated by ``authorize_model_for_key``."""
await authorize_model_for_key(model_id=model_id, llm_router=llm_router, user_api_key_dict=user_api_key_dict)
return get_credentials_for_model(
llm_router=llm_router,
model_id=model_id,
operation_context=operation_context,
)
def get_team_provider_credentials(
llm_router: Optional["Router"],
user_api_key_dict: "UserAPIKeyAuth",
@ -548,6 +594,25 @@ def add_internal_model_credentials(
data["_litellm_internal_model_credentials"] = MappingProxyType(dict(credentials))
def add_deployment_model_info(
data: dict,
llm_router: Optional["Router"],
model_id: str,
) -> None:
"""
Stamp the resolved deployment's `model_info` onto a direct (non-router) batch call
(in-place), the way the router does for routed calls, so the completed batch is
priced by its deployment id instead of the published model rate.
"""
deployment: Final = llm_router.get_credential_deployment(model_id=model_id) if llm_router is not None else None
if deployment is None:
return
data["litellm_metadata"] = {
**(data.get("litellm_metadata") or {}),
"model_info": deployment.model_info.model_dump(),
}
def prepare_data_with_credentials(
data: dict,
credentials: dict,
@ -573,21 +638,27 @@ def prepare_data_with_credentials(
data["file_id"] = file_id
def handle_model_based_routing(
async def handle_model_based_routing(
file_id: str,
request, # FastAPI Request object
llm_router, # Router instance
data: dict,
user_api_key_dict: "UserAPIKeyAuth",
check_file_id_encoding: bool = True,
) -> tuple[bool, str | None, str | None, dict | None]:
"""
Orchestrate model-based credential routing for file operations.
The model name comes from the caller (embedded in the file id, or a header, query
param or body field), so it is authorized against the caller's key, team, org and
project grants before any deployment credentials are resolved.
Args:
file_id: File ID (may contain embedded model info)
request: FastAPI request object
llm_router: LiteLLM Router instance
data: Request data dictionary
user_api_key_dict: The authenticated caller
check_file_id_encoding: Whether to check for embedded model in file_id
Returns:
@ -599,6 +670,7 @@ def handle_model_based_routing(
Raises:
HTTPException: If router unavailable or model not found
ProxyException: If the caller is not allowed to use the model
"""
model_from_id, model_from_param = extract_model_from_sources(
file_id=file_id,
@ -608,9 +680,10 @@ def handle_model_based_routing(
# Priority 1: Model embedded in file_id
if check_file_id_encoding and model_from_id is not None:
credentials = get_credentials_for_model(
credentials = await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=model_from_id,
user_api_key_dict=user_api_key_dict,
operation_context="file operation (file created with model)",
)
original_file_id: Final = get_original_file_id(file_id)
@ -618,9 +691,10 @@ def handle_model_based_routing(
# Priority 2: Model from header/query/body
elif model_from_param is not None:
credentials = get_credentials_for_model(
credentials = await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=model_from_param,
user_api_key_dict=user_api_key_dict,
operation_context="file operation",
)
return True, model_from_param, None, credentials

27
litellm/proxy/openai_files_endpoints/files_endpoints.py
View file

@ -69,7 +69,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
apply_team_provider_credentials,
encode_file_id_with_model,
extract_file_creation_params,
get_credentials_for_model,
get_authorized_credentials_for_model,
handle_model_based_routing,
prepare_data_with_credentials,
validate_file_list_limit,
@ -271,9 +271,10 @@ async def route_create_file(
# NEW: Handle model-based routing (no DB required)
if model is not None:
# Get credentials from model_list via router
credentials: Final = get_credentials_for_model(
credentials: Final = await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=model,
user_api_key_dict=user_api_key_dict,
operation_context="file upload",
)
@ -916,11 +917,12 @@ async def get_file_content(
model_used,
original_file_id,
credentials,
) = handle_model_based_routing(
) = await handle_model_based_routing(
file_id=file_id,
request=request,
llm_router=llm_router,
data=data,
user_api_key_dict=user_api_key_dict,
check_file_id_encoding=True,
)
@ -1131,15 +1133,16 @@ async def get_file(
model_used,
original_file_id,
credentials,
) = handle_model_based_routing(
) = await handle_model_based_routing(
file_id=file_id,
request=request,
llm_router=llm_router,
data=data,
user_api_key_dict=user_api_key_dict,
check_file_id_encoding=True,
)
if should_route:
if should_route and credentials is not None:
# Use model-based routing with credentials from config
prepare_data_with_credentials(
data=data,
@ -1148,7 +1151,10 @@ async def get_file(
include_internal_credentials=True,
)
response = await litellm.afile_retrieve(**data)
response = await litellm.afile_retrieve(
custom_llm_provider=credentials["custom_llm_provider"],
**data,
)
# Keep the encoded ID in response if it was originally encoded
if original_file_id and response and hasattr(response, "id") and response.id:
@ -1341,11 +1347,12 @@ async def delete_file(
model_used,
original_file_id,
credentials,
) = handle_model_based_routing(
) = await handle_model_based_routing(
file_id=file_id,
request=request,
llm_router=llm_router,
data=data,
user_api_key_dict=user_api_key_dict,
check_file_id_encoding=True,
)
@ -1534,11 +1541,12 @@ async def list_files(
response: Any | None = None
# Check for model-based credential routing (no file_id encoding check for list)
should_route, model_used, _, credentials = handle_model_based_routing(
should_route, model_used, _, credentials = await handle_model_based_routing(
file_id="", # No file_id for list endpoint
request=request,
llm_router=llm_router,
data=data,
user_api_key_dict=user_api_key_dict,
check_file_id_encoding=False,
)
@ -1565,9 +1573,10 @@ async def list_files(
status_code=500,
detail="LLM Router not initialized. Ensure models added to proxy.",
)
credentials = get_credentials_for_model(
credentials = await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=target_model_names_list[0],
user_api_key_dict=user_api_key_dict,
operation_context="file list",
)
prepare_data_with_credentials(data=data, credentials=credentials, include_internal_credentials=True)

45
litellm/proxy/vector_store_files_endpoints/endpoints.py
View file

@ -5,7 +5,6 @@ from fastapi.responses import ORJSONResponse
import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import _can_object_call_model, can_key_call_model
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.openai_endpoint_utils import (
@ -14,6 +13,8 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
get_custom_llm_provider_from_request_query,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
authorize_model_for_key,
get_credentials_for_model,
handle_model_based_routing,
prepare_data_with_credentials,
)
@ -144,11 +145,12 @@ async def _update_request_data_with_managed_file_id(
model_used,
original_file_id,
credentials,
) = handle_model_based_routing(
) = await handle_model_based_routing(
file_id=file_id,
request=request,
llm_router=llm_router,
data=data,
user_api_key_dict=user_api_key_dict,
check_file_id_encoding=True,
)
@ -210,26 +212,7 @@ async def _authorize_model_routing_hint(
) -> None:
if user_api_key_dict is None:
return
key_models: Final = getattr(user_api_key_dict, "models", None)
if not (isinstance(key_models, list) and "all-team-models" in key_models):
await can_key_call_model(
model=model,
llm_model_list=None,
valid_token=user_api_key_dict,
llm_router=llm_router,
)
team_models: Final = getattr(user_api_key_dict, "team_models", None)
if isinstance(team_models, list) and len(team_models) > 0:
_can_object_call_model(
model=model,
llm_router=llm_router,
models=team_models,
team_model_aliases=user_api_key_dict.team_model_aliases,
team_id=user_api_key_dict.team_id,
object_type="team",
)
await authorize_model_for_key(model_id=model, llm_router=llm_router, user_api_key_dict=user_api_key_dict)
async def _update_request_data_with_model_routing_hint(
@ -261,25 +244,15 @@ async def _update_request_data_with_model_routing_hint(
model_id=model_hint, team_id=caller_team_id
)
should_route = credentials is not None
else:
if isinstance(model_hint, str) and should_authorize_model_hint:
elif isinstance(model_hint, str):
if should_authorize_model_hint:
await _authorize_model_routing_hint(
model=model_hint,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
)
(
should_route,
_model_used,
_original_file_id,
credentials,
) = handle_model_based_routing(
file_id="",
request=request,
llm_router=llm_router,
data=data,
check_file_id_encoding=False,
)
credentials = get_credentials_for_model(llm_router=llm_router, model_id=model_hint)
should_route = True
if should_route and credentials is not None:
prepare_data_with_credentials(

2
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -2876,6 +2876,8 @@ class LiteLLMCompletionResponsesConfig:
cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
text_tokens=prompt_details.text_tokens,
audio_tokens=prompt_details.audio_tokens,
image_tokens=prompt_details.image_tokens,
video_tokens=prompt_details.video_tokens,
cached_tokens_details=(
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),

1
litellm/responses/utils.py
View file

@ -1194,6 +1194,7 @@ class ResponseAPILoggingUtils:
cached_tokens_details=getattr(
response_api_usage.input_tokens_details, "cached_tokens_details", None
),
video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None),
cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None),
google_maps_grounding_requests=getattr(

90
litellm/router.py
View file

@ -10313,6 +10313,55 @@ class Router:
return display_name
return None
def get_credential_deployment(self, model_id: str, team_id: str | None = None) -> Deployment | None:
"""
The deployment a passthrough endpoint (files, batches, etc.) resolves for a
model id or model name: by deployment id first, then by model_name, then by
the team's exact public model name, then by wildcard pattern (team wildcards
before global ones, so a global "openai/*" never shadows the team's own
entry). Name and wildcard lookups never resolve another team's deployment.
Returns None when nothing matches or the match is paused via
`LiteLLM_ProxyModelTable.blocked`, so callers cannot bypass an admin pause
by resolving the deployment directly.
"""
deployment: Final = (
self.get_deployment(model_id=model_id)
or self._get_model_group_deployment_usable_by_team(model_group_name=model_id, team_id=team_id)
or self._get_team_public_name_deployment(model_id=model_id, team_id=team_id)
or self._get_wildcard_deployment_usable_by_team(model_id=model_id, team_id=team_id)
)
if deployment is None or self._is_deployment_blocked(deployment):
return None
return deployment
def _get_team_public_name_deployment(self, model_id: str, team_id: str | None) -> Deployment | None:
if team_id is None:
return None
team_indices: Final = self.team_model_to_deployment_indices.get((team_id, model_id))
if not team_indices:
return None
team_model: Final = self.model_list[team_indices[0]]
return Deployment(**team_model) if isinstance(team_model, dict) else team_model
def _get_wildcard_deployment_usable_by_team(self, model_id: str, team_id: str | None) -> Deployment | None:
team_pattern_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None
team_wildcard_models: Final = team_pattern_router.route(model_id) if team_pattern_router else None
global_wildcard_models: Final = tuple(
wildcard_model
for wildcard_model in (self.pattern_router.route(model_id) or ())
if self._deployment_usable_by_team(wildcard_model, team_id)
)
potential_wildcard_models: Final = team_wildcard_models or global_wildcard_models
if not potential_wildcard_models:
return None
wildcard_deployment: Final = potential_wildcard_models[0]
if isinstance(wildcard_deployment, dict):
return Deployment(**wildcard_deployment)
if isinstance(wildcard_deployment, Deployment):
return wildcard_deployment
return None
def get_deployment_credentials_with_provider(
self, model_id: str, team_id: str | None = None
) -> dict[str, Any] | None:
@ -10320,8 +10369,8 @@ class Router:
Get API credentials and provider info from a model name in model_list.
Useful for passthrough endpoints (files, batches, etc.) that need credentials.
This method tries to find a deployment by model_id first, and if not found,
it tries to find by model_group_name (model_name).
Resolves the deployment with `get_credential_deployment` (by deployment id,
then model_name, team public model name, and wildcard pattern).
Args:
model_id: Model ID or model name from model_list (e.g., "gpt-4o-litellm")
@ -10342,43 +10391,8 @@ class Router:
credentials = router.get_deployment_credentials_with_provider("gpt-4o-litellm")
# Returns: {"api_key": "sk-...", "custom_llm_provider": "openai", "model": "gpt-4o", ...}
"""
# Try to get deployment by model_id first
deployment = self.get_deployment(model_id=model_id)
# If not found, try by model_group_name
deployment: Final = self.get_credential_deployment(model_id=model_id, team_id=team_id)
if deployment is None:
deployment = self._get_model_group_deployment_usable_by_team(model_group_name=model_id, team_id=team_id)
# If not found, check team-scoped deployments whose team public model
# name exactly matches model_id (wildcard team names are matched via
# team_pattern_routers below).
if deployment is None and team_id is not None:
team_indices: Final = self.team_model_to_deployment_indices.get((team_id, model_id), [])
if team_indices:
team_model: Final = self.model_list[team_indices[0]]
deployment = Deployment(**team_model) if isinstance(team_model, dict) else team_model
# If still not found, check for wildcard pattern matches. Team wildcard
# matches take priority so a global pattern (e.g. "openai/*") doesn't
# shadow the team's own entry.
if deployment is None:
team_pattern_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None
team_wildcard_models: Final = (team_pattern_router.route(model_id) or []) if team_pattern_router else []
global_wildcard_models: Final = [
wildcard_model
for wildcard_model in (self.pattern_router.route(model_id) or [])
if self._deployment_usable_by_team(wildcard_model, team_id)
]
potential_wildcard_models: Final = team_wildcard_models or global_wildcard_models
if potential_wildcard_models:
# Use the first matching wildcard deployment
deployment_dict: Final = potential_wildcard_models[0]
if isinstance(deployment_dict, dict):
deployment = Deployment(**deployment_dict)
elif isinstance(deployment_dict, Deployment):
deployment = deployment_dict
if deployment is None or self._is_deployment_blocked(deployment):
return None
# Get basic credentials

2
litellm/types/guardrails.py
View file

@ -819,7 +819,7 @@ class JavelinGuardrailConfigModel(BaseModel):
"""Configuration parameters for the Javelin guardrail"""
guard_name: str | None = Field(default=None, description="Name of the Javelin guard to use")
api_version: str | None = Field(default="v1", description="API version for Javelin service")
api_version: str | None = Field(default=None, description="API version for Javelin service")
metadata: dict | None = Field(default=None, description="Additional metadata to send with requests")
application: str | None = Field(default=None, description="Application name for Javelin service")
config: dict | None = Field(default=None, description="Additional configuration for the guardrail")

4
litellm/types/llms/openai.py
View file

@ -501,7 +501,7 @@ class CreateBatchRequest(TypedDict, total=False):
"""
completion_window: Literal["24h"]
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"]
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"]
input_file_id: str
metadata: dict[str, str] | None
output_expires_after: FileExpiresAfter
@ -1297,7 +1297,9 @@ class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
audio_tokens: int | None = None
cached_tokens: int = 0
cached_tokens_details: CachedTokensDetails | None = None
image_tokens: int | None = None
text_tokens: int | None = None
video_tokens: int | None = None
model_config = {"extra": "allow"}

5
litellm/types/utils.py
View file

@ -329,8 +329,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
output_cost_per_second_720p: ReadOnly[float | None]
output_cost_per_second_4k: ReadOnly[float | None]
ocr_cost_per_page: float | None # for OCR models
ocr_cost_per_page_batches: ReadOnly[float | None]
ocr_cost_per_credit: float | None # for OCR models priced by credit
annotation_cost_per_page: float | None # for OCR models
annotation_cost_per_page_batches: ReadOnly[float | None]
search_context_cost_per_query: SearchContextCostPerQuery | None # Cost for using web search tool
web_search_billing_unit: (
Literal["per_query", "per_prompt"] | None
@ -3669,8 +3671,10 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
output_cost_per_token_above_512k_tokens: float | None = None
output_vector_size: int | None = None
ocr_cost_per_page: float | None = None
ocr_cost_per_page_batches: float | None = None
ocr_cost_per_credit: float | None = None
annotation_cost_per_page: float | None = None
annotation_cost_per_page_batches: float | None = None
regional_processing_uplift_multiplier_eu: float | None = None
regional_processing_uplift_multiplier_us: float | None = None
regional_endpoint_uplift_multiplier: float | None = None
@ -4018,6 +4022,7 @@ class LlmProviders(str, Enum):
REDUCTO = "reducto"
RUNWAYML = "runwayml"
AWS_POLLY = "aws_polly"
TRANSCRIBE = "transcribe"
HUGGINGFACE = "huggingface"
TOGETHER_AI = "together_ai"
OPENROUTER = "openrouter"

26
litellm/utils.py
View file

@ -6118,8 +6118,10 @@ def _get_model_info_helper(
tpm=_model_info.get("tpm", None),
rpm=_model_info.get("rpm", None),
ocr_cost_per_page=_model_info.get("ocr_cost_per_page", None),
ocr_cost_per_page_batches=_model_info.get("ocr_cost_per_page_batches", None),
ocr_cost_per_credit=_model_info.get("ocr_cost_per_credit", None),
annotation_cost_per_page=_model_info.get("annotation_cost_per_page", None),
annotation_cost_per_page_batches=_model_info.get("annotation_cost_per_page_batches", None),
provider_specific_entry=_model_info.get("provider_specific_entry", None),
uses_embed_content=_model_info.get("uses_embed_content", None),
supports_image_size=_model_info.get("supports_image_size", None),
@ -8108,6 +8110,7 @@ def validate_chat_completion_user_messages(messages: list[AllMessageValues]):
def validate_chat_completion_tool_choice(
tool_choice: dict | str | None,
model: str = "",
) -> dict | str | None:
"""
Confirm the tool choice is passed in the OpenAI format.
@ -8123,12 +8126,19 @@ def validate_chat_completion_tool_choice(
# Standard OpenAI format: {"type": "function", "function": {...}}
if tool_choice.get("type") is None or tool_choice.get("function") is None:
raise Exception(
f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec"
raise BadRequestError(
message=f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec",
model=model,
llm_provider="",
)
return tool_choice
raise Exception(
f"Invalid tool choice, tool_choice={tool_choice}. Got={type(tool_choice)}. Expecting str, or dict. Please ensure tool_choice follows the OpenAI tool_choice spec"
raise BadRequestError(
message=(
f"Invalid tool choice, tool_choice={tool_choice}. Got={type(tool_choice)}. Expecting str, or dict. "
"Please ensure tool_choice follows the OpenAI tool_choice spec"
),
model=model,
llm_provider="",
)
@ -9114,6 +9124,10 @@ class ProviderConfigManager:
from litellm.llms.anthropic.files.transformation import AnthropicFilesConfig
return AnthropicFilesConfig()
elif LlmProviders.MISTRAL == provider:
from litellm.llms.mistral.files.transformation import MistralFilesConfig
return MistralFilesConfig()
return None
@staticmethod
@ -9125,6 +9139,10 @@ class ProviderConfigManager:
from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig
return BedrockBatchesConfig()
elif LlmProviders.MISTRAL == provider:
from litellm.llms.mistral.batches.transformation import MistralBatchesConfig
return MistralBatchesConfig()
return None
@staticmethod

40
model_prices_and_context_window.json
View file

@ -37476,51 +37476,66 @@
"mistral/mistral-ocr-latest": {
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.004,
"ocr_cost_per_page_batches": 0.002,
"annotation_cost_per_page": 0.005,
"annotation_cost_per_page_batches": 0.0025,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
"mistral/mistral-ocr-4-0": {
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.004,
"ocr_cost_per_page_batches": 0.002,
"annotation_cost_per_page": 0.005,
"annotation_cost_per_page_batches": 0.0025,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
"mistral/mistral-ocr-4-1": {
"annotation_cost_per_page": 0.005,
"annotation_cost_per_page_batches": 0.0025,
"litellm_provider": "mistral",
"mode": "ocr",
"ocr_cost_per_page": 0.004,
"ocr_cost_per_page_batches": 0.002,
"source": "https://docs.mistral.ai/models/model-cards/ocr-4-1",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
]
},
"mistral/mistral-ocr-2505-completion": {
"deprecation_date": "2026-05-31",
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.001,
"ocr_cost_per_page_batches": 0.0005,
"annotation_cost_per_page": 0.003,
"annotation_cost_per_page_batches": 0.0015,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
"mistral/mistral-ocr-2512": {
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.002,
"ocr_cost_per_page_batches": 0.001,
"annotation_cost_per_page": 0.003,
"annotation_cost_per_page_batches": 0.0015,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
@ -63741,31 +63756,40 @@
"mistral/mistral-ocr-3": {
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.002,
"ocr_cost_per_page_batches": 0.001,
"annotation_cost_per_page": 0.003,
"annotation_cost_per_page_batches": 0.0015,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
"mistral/mistral-ocr-3-0": {
"litellm_provider": "mistral",
"ocr_cost_per_page": 0.002,
"ocr_cost_per_page_batches": 0.001,
"annotation_cost_per_page": 0.003,
"annotation_cost_per_page_batches": 0.0015,
"mode": "ocr",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
],
"source": "https://mistral.ai/pricing#api-pricing"
},
"mistral/mistral-ocr-4": {
"annotation_cost_per_page": 0.005,
"annotation_cost_per_page_batches": 0.0025,
"litellm_provider": "mistral",
"mode": "ocr",
"ocr_cost_per_page": 0.004,
"ocr_cost_per_page_batches": 0.002,
"source": "https://docs.mistral.ai/models/model-cards/ocr-4-1",
"supported_endpoints": [
"/v1/ocr"
"/v1/ocr",
"/v1/batch"
]
},
"mistral/voxtral-mini-latest": {

8
model_prices_and_context_window.schema.json
View file

@ -53,6 +53,10 @@
"type": "number",
"minimum": 0
},
"annotation_cost_per_page_batches": {
"type": "number",
"minimum": 0
},
"audio_transcription_config": {
"type": "string"
},
@ -449,6 +453,10 @@
"type": "number",
"minimum": 0
},
"ocr_cost_per_page_batches": {
"type": "number",
"minimum": 0
},
"output_cost_per_audio_token": {
"type": "number",
"minimum": 0

2
provider_endpoints_support.json
View file

@ -1577,7 +1577,7 @@
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"batches": true,
"rerank": false,
"ocr": true,
"a2a": true,

1
tests/code_coverage_tests/recursive_detector.py
View file

@ -36,7 +36,6 @@ IGNORE_FUNCTIONS = [
"_collect_argument_paths", # max depth set.
"_split_text", # max depth set.
"_mask_sequence", # max depth set.
"_walk_payload", # max depth set (DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER).
"_delete_nested_value_custom", # max depth set (bounded by number of path segments).
"filter_exceptions_from_params", # max depth set (default 20) to prevent infinite recursion.
"__getattr__", # lazy loading pattern in litellm/__init__.py with proper caching to prevent infinite recursion.

6
tests/litellm_utils_tests/test_utils.py
View file

@ -1180,10 +1180,10 @@ def test_validate_chat_completion_tool_choice(tool_choice, expected_bool):
from litellm.utils import validate_chat_completion_tool_choice
if expected_bool:
validate_chat_completion_tool_choice(tool_choice=tool_choice)
validate_chat_completion_tool_choice(tool_choice=tool_choice, model="gpt-5.6-sol")
else:
with pytest.raises(Exception, match="Invalid tool choice"):
validate_chat_completion_tool_choice(tool_choice=tool_choice)
with pytest.raises(litellm.BadRequestError, match="Invalid tool choice"):
validate_chat_completion_tool_choice(tool_choice=tool_choice, model="gpt-5.6-sol")
def test_models_by_provider():

80
tests/litellm_utils_tests/test_validate_tool_choice.py
View file

@ -1,60 +1,74 @@
import re
from typing import Final
import pytest
import litellm
from litellm.utils import validate_chat_completion_tool_choice
MODEL: Final = "anthropic/claude-haiku-4-5"
def test_validate_tool_choice_none():
"""Test that None is returned as-is."""
result = validate_chat_completion_tool_choice(None)
result = validate_chat_completion_tool_choice(None, model=MODEL)
assert result is None
def test_validate_tool_choice_string():
"""Test that string values are returned as-is."""
assert validate_chat_completion_tool_choice("auto") == "auto"
assert validate_chat_completion_tool_choice("none") == "none"
assert validate_chat_completion_tool_choice("required") == "required"
assert validate_chat_completion_tool_choice("auto", model=MODEL) == "auto"
assert validate_chat_completion_tool_choice("none", model=MODEL) == "none"
assert validate_chat_completion_tool_choice("required", model=MODEL) == "required"
def test_validate_tool_choice_standard_dict():
"""Test standard OpenAI format with function."""
tool_choice = {"type": "function", "function": {"name": "my_function"}}
result = validate_chat_completion_tool_choice(tool_choice)
result = validate_chat_completion_tool_choice(tool_choice, model=MODEL)
assert result == tool_choice
def test_validate_tool_choice_cursor_format():
"""Cursor IDE format {"type": "auto"} is unwrapped to the bare string."""
assert validate_chat_completion_tool_choice({"type": "auto"}) == "auto"
assert validate_chat_completion_tool_choice({"type": "none"}) == "none"
assert validate_chat_completion_tool_choice({"type": "required"}) == "required"
assert validate_chat_completion_tool_choice({"type": "auto"}, model=MODEL) == "auto"
assert validate_chat_completion_tool_choice({"type": "none"}, model=MODEL) == "none"
assert validate_chat_completion_tool_choice({"type": "required"}, model=MODEL) == "required"
def test_validate_tool_choice_invalid_dict():
"""Test that invalid dict formats raise exceptions."""
# Missing both type and function
with pytest.raises(Exception, match='Invalid tool choice, tool_choice=\\{\\}\\. Please ensure') as exc_info:
validate_chat_completion_tool_choice({})
assert "Invalid tool choice" in str(exc_info.value)
# Invalid type value
with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\{'type': 'invalid'\\}\\.") as exc_info:
validate_chat_completion_tool_choice({"type": "invalid"})
assert "Invalid tool choice" in str(exc_info.value)
# Has type but missing function when type is "function"
with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\{'type': 'function'\\}\\.") as exc_info:
validate_chat_completion_tool_choice({"type": "function"})
assert "Invalid tool choice" in str(exc_info.value)
@pytest.mark.parametrize(
"tool_choice",
[
{},
{"type": "invalid"},
{"type": "function"},
{"name": "lookup_fruit"},
{"type": "file_search"},
],
)
def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice):
"""A dict shape chat completions cannot carry is the caller's mistake: a 400 that names the field, never a 500."""
with pytest.raises(
litellm.BadRequestError, match=f"Invalid tool choice, tool_choice={re.escape(str(tool_choice))}\\. Please ensure"
) as exc_info:
validate_chat_completion_tool_choice(tool_choice, model=MODEL)
assert exc_info.value.status_code == 400
assert exc_info.value.model == MODEL
def test_validate_tool_choice_invalid_type():
"""Test that invalid types raise exceptions."""
with pytest.raises(Exception, match="<class 'int'>\\. Expecting str, or dict\\. Please ensure") as exc_info:
validate_chat_completion_tool_choice(123)
assert "Got=<class 'int'>" in str(exc_info.value)
@pytest.mark.parametrize("tool_choice", [123, []])
def test_validate_tool_choice_invalid_type_is_a_400(tool_choice):
"""A non-str, non-dict tool_choice is rejected as a 400 that names the type it got."""
with pytest.raises(
litellm.BadRequestError, match=f"Got={re.escape(str(type(tool_choice)))}\\. Expecting str, or dict\\."
) as exc_info:
validate_chat_completion_tool_choice(tool_choice, model=MODEL)
assert exc_info.value.status_code == 400
with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\[\\]\\. Got=<class 'list'>\\.") as exc_info:
validate_chat_completion_tool_choice([])
assert "Got=<class 'list'>" in str(exc_info.value)
def test_validate_tool_choice_without_model_is_still_a_400():
"""Callers that predate the model argument keep getting a 400, with an empty model on the error."""
with pytest.raises(litellm.BadRequestError, match="Invalid tool choice") as exc_info:
validate_chat_completion_tool_choice({"type": "bogus"})
assert exc_info.value.status_code == 400
assert exc_info.value.model == ""

162
tests/test_litellm/batches/test_batch_utils.py
View file

@ -647,9 +647,7 @@ async def test_calculate_vertex_disable_transform_needs_model_name(monkeypatch):
lambda content, model: pytest.fail("raw vertex path should not run"),
)
result = await bu.calculate_batch_cost_and_usage(
file_content_dictionary=[], custom_llm_provider="vertex_ai"
)
result = await bu.calculate_batch_cost_and_usage(file_content_dictionary=[], custom_llm_provider="vertex_ai")
assert result.cost == 0.0
assert result.usage.total_tokens == 0
assert result.models == []
@ -1284,6 +1282,7 @@ async def test_handle_completed_batch_no_output_file_is_zero(monkeypatch):
result set - zero cost, zero usage, no models - instead of letting the file
fetch raise "Output file id is None" on every aretrieve_batch logging poll.
"""
# The output-file fetch must not even be attempted when there is no output file.
async def _must_not_fetch(*args, **kwargs):
pytest.fail("_fetch_batch_output_file_content should not be called")
@ -1410,7 +1409,10 @@ def test_anthropic_response_body_is_result_message():
def test_anthropic_usage_conversion_includes_cache_tokens():
body = {"model": "claude-sonnet-4-5-20250929", "usage": _anthropic_usage(1000, 200, cache_creation=2000, cache_read=8000)}
body = {
"model": "claude-sonnet-4-5-20250929",
"usage": _anthropic_usage(1000, 200, cache_creation=2000, cache_read=8000),
}
usage = bu._get_batch_job_usage_from_response_body(body, custom_llm_provider="anthropic")
assert usage.prompt_tokens == 11000
assert usage.completion_tokens == 200
@ -1425,7 +1427,9 @@ def test_bedrock_model_output_line_success_check():
"modelOutput": {"model": "claude-sonnet-4-6", "usage": {"input_tokens": 13, "output_tokens": 5}},
}
assert bu._batch_response_was_successful(row, custom_llm_provider="bedrock") is True
assert bu._get_response_from_batch_job_output_file(row, custom_llm_provider="bedrock")["model"] == "claude-sonnet-4-6"
assert (
bu._get_response_from_batch_job_output_file(row, custom_llm_provider="bedrock")["model"] == "claude-sonnet-4-6"
)
def test_bedrock_cost_uses_deployment_model_name():
@ -1479,7 +1483,13 @@ def test_total_usage_without_cache_tokens_has_no_prompt_details(monkeypatch):
rows = [
{
"custom_id": "req-1",
"response": {"status_code": 200, "body": {"model": "gpt-5.2", "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}},
"response": {
"status_code": 200,
"body": {
"model": "gpt-5.2",
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
},
}
]
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
@ -1521,7 +1531,9 @@ def test_anthropic_cost_without_model_info_uses_batch_cost_calculator(monkeypatc
lambda **kw: pytest.fail("anthropic rows must not go through completion_cost"),
)
result = bu._aggregate_batch_cost_usage_models(entries=[_anthropic_succeeded_row()], custom_llm_provider="anthropic")
result = bu._aggregate_batch_cost_usage_models(
entries=[_anthropic_succeeded_row()], custom_llm_provider="anthropic"
)
assert result.cost == pytest.approx(0.3)
assert seen[0]["model"] == "claude-sonnet-4-5-20250929"
@ -1556,7 +1568,11 @@ async def test_calculate_batch_cost_and_usage_anthropic_end_to_end():
)
assert result.cost == pytest.approx(1000 * 3e-6 / 2 + 8000 * 3e-7 / 2 + 2000 * 3.75e-6 / 2 + 200 * 15e-6 / 2)
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (11000, 200, 11200)
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (
11000,
200,
11200,
)
assert result.models == ["claude-sonnet-4-5"]
@ -1721,7 +1737,10 @@ async def test_handle_completed_batch_honors_deployment_pricing(monkeypatch) ->
def test_bedrock_converse_shaped_batch_usage_is_parsed():
body = {"model": "us.amazon.nova-lite-v1:0", "usage": {"inputTokens": 2202, "outputTokens": 540, "totalTokens": 2742}}
body = {
"model": "us.amazon.nova-lite-v1:0",
"usage": {"inputTokens": 2202, "outputTokens": 540, "totalTokens": 2742},
}
usage = bu._get_batch_job_usage_from_response_body(body, custom_llm_provider="bedrock")
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (2202, 540, 2742)
@ -1809,6 +1828,7 @@ def test_unparsable_bedrock_batch_usage_warns(caplog):
# batch_cost_is_final
# --------------------------------------------------------------------------- #
def _retrieved_batch(
status: str, output_file_id: str | None = None, counts: BatchRequestCounts | None = None
) -> LiteLLMBatch:
@ -1857,3 +1877,127 @@ class TestBatchCostIsFinal:
@pytest.mark.parametrize("status", ["failed", "expired", "cancelled"])
def test_other_terminal_statuses_are_final(self, status):
assert bu.batch_cost_is_final(_retrieved_batch(status)) is True
def _ocr_row(pages_processed, annotation_pages=None, model="mistral-ocr-latest"):
usage_info = {"pages_processed": pages_processed, "doc_size_bytes": 4096}
if annotation_pages is not None:
usage_info["pages_processed_annotation"] = annotation_pages
return _success_row(
model=model, pages=[{"index": i, "markdown": "x"} for i in range(pages_processed)], usage_info=usage_info
)
def test_ocr_rows_are_priced_per_page_at_batch_rate(monkeypatch):
monkeypatch.setattr(
litellm,
"get_model_info",
lambda model, custom_llm_provider=None: {"ocr_cost_per_page": 0.004, "ocr_cost_per_page_batches": 0.002},
)
result = bu._aggregate_batch_cost_usage_models(
entries=[_ocr_row(3), _ocr_row(5), _failed_row(model="mistral-ocr-latest")],
custom_llm_provider="mistral",
model_name="mistral/mistral-ocr-latest",
)
assert result.cost == pytest.approx(8 * 0.002)
assert result.prompt_cost == pytest.approx(8 * 0.002)
assert result.completion_cost == 0.0
assert (result.successful_requests, result.failed_requests) == (2, 1)
assert result.usage.total_tokens == 0
assert result.models == ["mistral/mistral-ocr-latest"]
def test_ocr_rows_fall_back_to_sync_page_rate_without_batch_price(monkeypatch):
monkeypatch.setattr(litellm, "get_model_info", lambda model, custom_llm_provider=None: {"ocr_cost_per_page": 0.004})
result = bu._aggregate_batch_cost_usage_models(entries=[_ocr_row(2)], custom_llm_provider="mistral")
assert result.cost == pytest.approx(2 * 0.004)
def test_ocr_rows_bill_annotation_pages_separately(monkeypatch):
monkeypatch.setattr(
litellm,
"get_model_info",
lambda model, custom_llm_provider=None: {
"ocr_cost_per_page_batches": 0.002,
"annotation_cost_per_page_batches": 0.0025,
},
)
result = bu._aggregate_batch_cost_usage_models(
entries=[_ocr_row(4, annotation_pages=4)], custom_llm_provider="mistral"
)
assert result.cost == pytest.approx(4 * 0.002 + 4 * 0.0025)
def test_ocr_rows_use_deployment_model_info_pricing_over_cost_map(monkeypatch):
monkeypatch.setattr(
litellm, "get_model_info", lambda model, custom_llm_provider=None: pytest.fail("cost map must not be consulted")
)
result = bu._aggregate_batch_cost_usage_models(
entries=[_ocr_row(10)],
custom_llm_provider="mistral",
model_info={"ocr_cost_per_page_batches": 0.001},
)
assert result.cost == pytest.approx(0.01)
def test_ocr_rows_keep_the_published_page_rate_when_the_deployment_prices_only_annotations(monkeypatch):
monkeypatch.setattr(
litellm,
"get_model_info",
lambda model, custom_llm_provider=None: {
"ocr_cost_per_page_batches": 0.002,
"annotation_cost_per_page_batches": 0.0025,
},
)
result = bu._aggregate_batch_cost_usage_models(
entries=[_ocr_row(4, annotation_pages=4)],
custom_llm_provider="mistral",
model_info={"annotation_cost_per_page_batches": 0.01},
)
assert result.cost == pytest.approx(4 * 0.002 + 4 * 0.01)
def test_ocr_rows_keep_the_deployment_page_rate_when_the_unmapped_model_has_no_annotation_price(monkeypatch):
def _unmapped(model, custom_llm_provider=None):
raise Exception(f"This model isn't mapped yet: {model}")
monkeypatch.setattr(litellm, "get_model_info", _unmapped)
result = bu._aggregate_batch_cost_usage_models(
entries=[_ocr_row(4, annotation_pages=4, model="my-private-ocr-model")],
custom_llm_provider="mistral",
model_info={"ocr_cost_per_page_batches": 0.001},
)
assert result.cost == pytest.approx(4 * 0.001 + 4 * 0.001)
def test_ocr_rows_bill_the_deployment_sync_page_rate_over_the_published_batch_rate(monkeypatch):
monkeypatch.setattr(
litellm, "get_model_info", lambda model, custom_llm_provider=None: pytest.fail("cost map must not be consulted")
)
result = bu._aggregate_batch_cost_usage_models(
entries=[_ocr_row(3)],
custom_llm_provider="mistral",
model_info={"ocr_cost_per_page": 0.0912},
)
assert result.cost == pytest.approx(3 * 0.0912)
def test_ocr_rows_without_pricing_bill_zero_but_count_as_successful(monkeypatch):
monkeypatch.setattr(litellm, "get_model_info", lambda model, custom_llm_provider=None: {"mode": "ocr"})
result = bu._aggregate_batch_cost_usage_models(entries=[_ocr_row(3)], custom_llm_provider="mistral")
assert result.cost == 0.0
assert (result.successful_requests, result.failed_requests) == (1, 0)
def test_chat_rows_from_mistral_still_use_token_pricing(monkeypatch):
monkeypatch.setattr(
litellm,
"get_model_info",
lambda model, custom_llm_provider=None: {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002},
)
result = bu._aggregate_batch_cost_usage_models(
entries=[_success_row(model="mistral-small-latest", usage=_usage(10, 5))],
custom_llm_provider="mistral",
)
assert result.cost == pytest.approx((10 * 0.001 + 5 * 0.002) / 2)
assert result.usage.total_tokens == 15

100
tests/test_litellm/batches/test_main.py
View file

@ -66,9 +66,7 @@ def seams():
stack.enter_context(patch.object(bm, "openai_batches_instance", openai_i))
stack.enter_context(patch.object(bm, "azure_batches_instance", azure_i))
stack.enter_context(patch.object(bm, "vertex_ai_batches_instance", vertex_i))
stack.enter_context(
patch.object(bm, "anthropic_batches_instance", anthropic_i)
)
stack.enter_context(patch.object(bm, "anthropic_batches_instance", anthropic_i))
stack.enter_context(patch.object(bm, "base_llm_http_handler", base_http))
stack.enter_context(patch.object(bm, "BedrockBatchesHandler", bedrock_arn))
yield Seams(
@ -174,9 +172,7 @@ def test_create__provider_config_routes_to_base_http_handler(seams):
"get_provider_batches_config",
return_value=MagicMock(name="provider_config"),
):
result = bm.create_batch(
**CREATE_KW, custom_llm_provider="bedrock", model="bedrock/my-batch-model"
)
result = bm.create_batch(**CREATE_KW, custom_llm_provider="bedrock", model="bedrock/my-batch-model")
assert result is seams.base_http.create_batch.return_value
_assert_only(seams.base_http.create_batch, seams, "create_batch")
@ -281,9 +277,7 @@ def test_retrieve__bedrock_model_invocation_job_arn(seams):
result = bm.retrieve_batch(batch_id=arn, custom_llm_provider="bedrock")
seams.bedrock_arn._handle_model_invocation_job_status.assert_called_once()
assert (
result is seams.bedrock_arn._handle_model_invocation_job_status.return_value
)
assert result is seams.bedrock_arn._handle_model_invocation_job_status.return_value
seams.bedrock_arn._handle_async_invoke_status.assert_not_called()
@ -385,9 +379,7 @@ def test_cancel__unsupported_provider_raises_badrequest(seams):
def test_cancel__async_flag_propagates_is_async(seams):
bm.cancel_batch(
batch_id="batch-1", custom_llm_provider="openai", acancel_batch=True
)
bm.cancel_batch(batch_id="batch-1", custom_llm_provider="openai", acancel_batch=True)
assert seams.openai.cancel_batch.call_args.kwargs["_is_async"] is True
@ -415,9 +407,7 @@ async def test_acreate_batch_delegates_to_create_batch():
@pytest.mark.asyncio
async def test_aretrieve_batch_delegates_to_retrieve_batch():
with patch.object(bm, "retrieve_batch", MagicMock(return_value="SENTINEL")) as m:
result = await bm.aretrieve_batch(
batch_id="batch-1", custom_llm_provider="azure"
)
result = await bm.aretrieve_batch(batch_id="batch-1", custom_llm_provider="azure")
assert result == "SENTINEL"
assert m.call_count == 1
@ -429,9 +419,7 @@ async def test_aretrieve_batch_delegates_to_retrieve_batch():
@pytest.mark.asyncio
async def test_alist_batches_delegates_to_list_batches():
with patch.object(bm, "list_batches", MagicMock(return_value="SENTINEL")) as m:
result = await bm.alist_batches(
after="cur", limit=3, custom_llm_provider="vertex_ai"
)
result = await bm.alist_batches(after="cur", limit=3, custom_llm_provider="vertex_ai")
assert result == "SENTINEL"
assert m.call_count == 1
@ -444,9 +432,7 @@ async def test_alist_batches_delegates_to_list_batches():
@pytest.mark.asyncio
async def test_acancel_batch_delegates_to_cancel_batch():
with patch.object(bm, "cancel_batch", MagicMock(return_value="SENTINEL")) as m:
result = await bm.acancel_batch(
batch_id="batch-1", custom_llm_provider="openai"
)
result = await bm.acancel_batch(batch_id="batch-1", custom_llm_provider="openai")
assert result == "SENTINEL"
assert m.call_count == 1
@ -499,9 +485,7 @@ def _sent(mock_method, *keys):
def test_create__openai_credentials_passthrough(seams):
bm.create_batch(**CREATE_KW, custom_llm_provider="openai", **OPENAI_CREDS)
assert _sent(
seams.openai.create_batch, "api_key", "api_base", "organization", "max_retries"
) == {
assert _sent(seams.openai.create_batch, "api_key", "api_base", "organization", "max_retries") == {
"api_key": "sk-user-openai",
"api_base": "https://openai.user.test",
"organization": "org-user-123",
@ -512,9 +496,7 @@ def test_create__openai_credentials_passthrough(seams):
def test_create__azure_credentials_passthrough(seams):
bm.create_batch(**CREATE_KW, custom_llm_provider="azure", **AZURE_CREDS)
assert _sent(
seams.azure.create_batch, "api_key", "api_base", "api_version"
) == {
assert _sent(seams.azure.create_batch, "api_key", "api_base", "api_version") == {
"api_key": "sk-user-azure",
"api_base": "https://azure.user.test",
"api_version": "2024-12-99",
@ -564,9 +546,7 @@ def test_create__provider_config_credentials_passthrough(seams):
def test_retrieve__openai_credentials_passthrough(seams):
bm.retrieve_batch(batch_id="b1", custom_llm_provider="openai", **OPENAI_CREDS)
assert _sent(
seams.openai.retrieve_batch, "api_key", "api_base", "organization"
) == {
assert _sent(seams.openai.retrieve_batch, "api_key", "api_base", "organization") == {
"api_key": "sk-user-openai",
"api_base": "https://openai.user.test",
"organization": "org-user-123",
@ -576,9 +556,7 @@ def test_retrieve__openai_credentials_passthrough(seams):
def test_retrieve__azure_credentials_passthrough(seams):
bm.retrieve_batch(batch_id="b1", custom_llm_provider="azure", **AZURE_CREDS)
assert _sent(
seams.azure.retrieve_batch, "api_key", "api_base", "api_version"
) == {
assert _sent(seams.azure.retrieve_batch, "api_key", "api_base", "api_version") == {
"api_key": "sk-user-azure",
"api_base": "https://azure.user.test",
"api_version": "2024-12-99",
@ -640,9 +618,7 @@ def test_retrieve__provider_config_credentials_passthrough(seams):
def test_list__openai_credentials_passthrough(seams):
bm.list_batches(custom_llm_provider="openai", **OPENAI_CREDS)
assert _sent(
seams.openai.list_batches, "api_key", "api_base", "organization"
) == {
assert _sent(seams.openai.list_batches, "api_key", "api_base", "organization") == {
"api_key": "sk-user-openai",
"api_base": "https://openai.user.test",
"organization": "org-user-123",
@ -652,9 +628,7 @@ def test_list__openai_credentials_passthrough(seams):
def test_list__azure_credentials_passthrough(seams):
bm.list_batches(custom_llm_provider="azure", **AZURE_CREDS)
assert _sent(
seams.azure.list_batches, "api_key", "api_base", "api_version"
) == {
assert _sent(seams.azure.list_batches, "api_key", "api_base", "api_version") == {
"api_key": "sk-user-azure",
"api_base": "https://azure.user.test",
"api_version": "2024-12-99",
@ -682,9 +656,7 @@ def test_list__vertex_credentials_passthrough(seams):
def test_cancel__openai_credentials_passthrough(seams):
bm.cancel_batch(batch_id="b1", custom_llm_provider="openai", **OPENAI_CREDS)
assert _sent(
seams.openai.cancel_batch, "api_key", "api_base", "organization"
) == {
assert _sent(seams.openai.cancel_batch, "api_key", "api_base", "organization") == {
"api_key": "sk-user-openai",
"api_base": "https://openai.user.test",
"organization": "org-user-123",
@ -694,9 +666,7 @@ def test_cancel__openai_credentials_passthrough(seams):
def test_cancel__azure_credentials_passthrough(seams):
bm.cancel_batch(batch_id="b1", custom_llm_provider="azure", **AZURE_CREDS)
assert _sent(
seams.azure.cancel_batch, "api_key", "api_base", "api_version"
) == {
assert _sent(seams.azure.cancel_batch, "api_key", "api_base", "api_version") == {
"api_key": "sk-user-azure",
"api_base": "https://azure.user.test",
"api_version": "2024-12-99",
@ -778,3 +748,43 @@ def test_retrieve__omits_trusted_model_credentials_when_not_supplied(seams):
litellm_params = logging_obj.update_from_kwargs.call_args.kwargs["litellm_params"]
assert "_litellm_internal_model_credentials" not in litellm_params
# =========================================================================== #
# mistral - a provider-config provider, like bedrock, so it requires `model`
# =========================================================================== #
def test_create__mistral_ocr_routes_to_base_http_handler_with_mistral_config(seams):
result = bm.create_batch(
completion_window="24h",
endpoint="/v1/ocr",
input_file_id="file-abc",
custom_llm_provider="mistral",
model="mistral/mistral-ocr-latest",
)
assert result is seams.base_http.create_batch.return_value
_assert_only(seams.base_http.create_batch, seams, "create_batch")
forwarded = seams.base_http.create_batch.call_args.kwargs
assert type(forwarded["provider_config"]).__name__ == "MistralBatchesConfig"
assert forwarded["model"] == "mistral-ocr-latest"
assert forwarded["create_batch_data"]["endpoint"] == "/v1/ocr"
def test_create__mistral_without_model_raises_badrequest(seams):
with pytest.raises(litellm.exceptions.BadRequestError):
bm.create_batch(**CREATE_KW, custom_llm_provider="mistral")
for m in _all_seam_methods(seams, "create_batch"):
m.assert_not_called()
def test_retrieve__mistral_routes_to_base_http_handler_with_mistral_config(seams):
result = bm.retrieve_batch(batch_id="job-1", custom_llm_provider="mistral", model="mistral/mistral-ocr-latest")
assert result is seams.base_http.retrieve_batch.return_value
_assert_only(seams.base_http.retrieve_batch, seams, "retrieve_batch")
forwarded = seams.base_http.retrieve_batch.call_args.kwargs
assert type(forwarded["provider_config"]).__name__ == "MistralBatchesConfig"
assert forwarded["batch_id"] == "job-1"

12
tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py
View file

@ -10,6 +10,7 @@ import pytest
import litellm
from litellm.caching.caching import DualCache
from litellm.integrations.SlackAlerting.budget_alert_types import get_budget_alert_type
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.proxy._types import CallInfo, Litellm_EntityType
from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingCacheKeys
@ -61,9 +62,8 @@ class TestSlackAlerting(unittest.TestCase):
self.assertNotIn("*token:*", result)
def test_get_event_and_event_message_max_budget(self):
# Initial setup with no event
event = None
event_message = "Test Message: "
event_message = get_budget_alert_type("user_budget").get_event_message()
# Test case 1: When spend exceeds max_budget
user_info = CallInfo(
@ -78,7 +78,7 @@ class TestSlackAlerting(unittest.TestCase):
self.assertEqual(event, "budget_crossed")
self.assertTrue("Budget Crossed" in event_message)
# Test case 2: When 5% of max_budget is left
event_message = get_budget_alert_type("user_budget").get_event_message()
user_info = CallInfo(
max_budget=100.0,
spend=95.0,
@ -89,9 +89,9 @@ class TestSlackAlerting(unittest.TestCase):
user_info=user_info, event=event, event_message=event_message
)
self.assertEqual(event, "threshold_crossed")
self.assertTrue("5% Threshold Crossed" in event_message)
self.assertEqual(event_message, "User Budget: 5% or less of budget remaining")
# Test case 3: When 15% of max_budget is left
event_message = get_budget_alert_type("user_budget").get_event_message()
user_info = CallInfo(
max_budget=100.0,
spend=85.0,
@ -102,7 +102,7 @@ class TestSlackAlerting(unittest.TestCase):
user_info=user_info, event=event, event_message=event_message
)
self.assertEqual(event, "threshold_crossed")
self.assertTrue("15% Threshold Crossed" in event_message)
self.assertEqual(event_message, "User Budget: 15% or less of budget remaining")
def test_get_event_and_event_message_soft_budget(self):
# Initial setup with no event

95
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -17,12 +17,14 @@ from openai._legacy_response import HttpxBinaryResponseContent
import litellm
from litellm._logging import session_id_var, trace_id_var
from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST
from litellm.cost_calculator import ocr_batch_cost
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.litellm_core_utils.litellm_logging import (
_get_status_fields,
set_callbacks,
)
from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.utils import (
CallTypes,
@ -59,6 +61,16 @@ def test_get_masked_api_base(logging_obj):
assert type(masked_api_base) == str
def test_pre_call_tolerates_missing_api_base(logging_obj):
"""Presigned batch retrieves (Mistral, Bedrock) build their own URL and pass api_base=None
to pre_call; masking must not raise or the request's pre-call logging is silently lost."""
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
logging_obj.pre_call(input="", api_key="", additional_args={"api_base": None, "headers": {}})
assert logging_obj.model_call_details["litellm_params"]["api_base"] == ""
def test_post_call_serializes_dict_with_datetime(logging_obj):
import datetime
@ -519,6 +531,36 @@ class TestGetRouterDeploymentModelInfo:
finally:
litellm.model_cost.pop(deployment_id, None)
def test_ocr_only_deployment_pricing_reaches_batch_ocr_cost(self, logging_obj) -> None:
"""Regression: a deployment priced only per page was treated as unpriced, so a retrieved OCR batch
billed at the published rate while the same deployment's synchronous OCR calls billed at its own."""
deployment_id = "deploy-ocr-only-pricing-1"
litellm.model_cost[deployment_id] = {
"id": deployment_id,
"litellm_provider": "mistral",
"mode": "ocr",
"ocr_cost_per_page": 0.0456,
"ocr_cost_per_page_batches": 0.0123,
}
logging_obj.litellm_params = {
"litellm_metadata": {"model_info": {"id": deployment_id}},
"model": "mistral/mistral-ocr-latest",
}
logging_obj.model_call_details["model"] = "mistral/mistral-ocr-latest"
published_annotation_rate = litellm.model_cost["mistral/mistral-ocr-latest"]["annotation_cost_per_page_batches"]
try:
info = logging_obj.get_router_deployment_model_info()
assert info is not None
assert info["ocr_cost_per_page_batches"] == 0.0123
pages_only = OCRUsageInfo(pages_processed=3)
assert ocr_batch_cost("mistral-ocr-latest", "mistral", pages_only, info)[0] == pytest.approx(3 * 0.0123)
with_annotations = OCRUsageInfo(pages_processed=3, pages_processed_annotation=2)
assert ocr_batch_cost("mistral-ocr-latest", "mistral", with_annotations, info)[0] == pytest.approx(
3 * 0.0123 + 2 * published_annotation_rate
)
finally:
litellm.model_cost.pop(deployment_id, None)
class TestRetrieveBatchCostPassesModelIdentity:
"""Regression: retrieving a batch priced it with no model identity at all.
@ -3958,9 +4000,7 @@ def test_get_standard_logging_object_payload_carries_matched_access_groups(loggi
"model": "gpt-4o",
"messages": [],
"litellm_params": {
"metadata": {
"user_api_key_matched_model_access_groups": ["premium-pool", "shared-pool"]
},
"metadata": {"user_api_key_matched_model_access_groups": ["premium-pool", "shared-pool"]},
"proxy_server_request": {"body": {}},
},
},
@ -4044,9 +4084,7 @@ def _model_router_response(selected_model: str, stamp: bool):
from litellm.types.utils import ModelResponse
response = ModelResponse(model=selected_model)
response._hidden_params = (
{AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model} if stamp else {}
)
response._hidden_params = {AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model} if stamp else {}
return response
@ -4070,9 +4108,7 @@ def test_standard_logging_payload_uses_stamped_model_router_model(logging_obj):
"messages": [],
"litellm_params": {"metadata": {}},
},
init_response_obj=_model_router_response(
"azure_ai/grok-4-1-fast-reasoning", stamp=True
),
init_response_obj=_model_router_response("azure_ai/grok-4-1-fast-reasoning", stamp=True),
start_time=now,
end_time=now,
logging_obj=logging_obj,
@ -4104,9 +4140,7 @@ def test_standard_logging_payload_keeps_requested_model_without_router_stamp(
"messages": [],
"litellm_params": {"metadata": {}},
},
init_response_obj=_model_router_response(
"azure_ai/grok-4-1-fast-reasoning", stamp=False
),
init_response_obj=_model_router_response("azure_ai/grok-4-1-fast-reasoning", stamp=False),
start_time=now,
end_time=now,
logging_obj=logging_obj,
@ -5594,9 +5628,7 @@ class TestNonInferenceCallTypesAreNotBilled:
init_response_obj=self._retrieved_response(),
start_time=now,
end_time=now,
logging_obj=self._logging_obj(
"aget_responses", litellm_metadata=self.BACKGROUND_POLL_METADATA
),
logging_obj=self._logging_obj("aget_responses", litellm_metadata=self.BACKGROUND_POLL_METADATA),
status="success",
)
@ -5842,9 +5874,7 @@ async def test_streaming_success_callbacks_survive_cost_calculation_failure():
releasing.async_log_success_event = AsyncMock()
patcher, logging_obj = _streaming_logging_obj_with_callbacks([releasing])
with patcher, patch.object(
logging_obj, "_response_cost_calculator", side_effect=ValueError("bad usage block")
):
with patcher, patch.object(logging_obj, "_response_cost_calculator", side_effect=ValueError("bad usage block")):
await logging_obj.async_success_handler(result=_assembled_stream_result())
assert logging_obj.model_call_details["response_cost"] is None
@ -5857,8 +5887,9 @@ async def test_streaming_success_callbacks_survive_standard_logging_payload_fail
releasing.async_log_success_event = AsyncMock()
patcher, logging_obj = _streaming_logging_obj_with_callbacks([releasing])
with patcher, patch.object(
logging_obj, "_build_standard_logging_payload", side_effect=ValueError("incomplete stream")
with (
patcher,
patch.object(logging_obj, "_build_standard_logging_payload", side_effect=ValueError("incomplete stream")),
):
await logging_obj.async_success_handler(result=_assembled_stream_result())
@ -6206,6 +6237,8 @@ def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_pa
)
for hook in [cb for cb in litellm.callbacks if isinstance(cb, VectorStorePreCallHook)]:
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, hook)
def test_newrelic_dispatch_prefers_otel_v2_when_flag_on(monkeypatch):
"""With LITELLM_OTEL_V2 on and operator credentials present, the "newrelic"
callback builds the OTel v2 logger (per-team credential routing); with the
@ -6361,7 +6394,9 @@ def test_get_error_information_skips_traceback_for_budget_rejection_with_provide
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
assert litellm.log_client_error_tracebacks is False
over_budget = _raise_and_catch(litellm.BudgetExceededError(current_cost=0.01, max_budget=0.0, llm_provider="anthropic"))
over_budget = _raise_and_catch(
litellm.BudgetExceededError(current_cost=0.01, max_budget=0.0, llm_provider="anthropic")
)
result = StandardLoggingPayloadSetup.get_error_information(over_budget)
assert result["error_code"] == "429"
assert result["llm_provider"] == "anthropic"
@ -6934,9 +6969,7 @@ def test_passthrough_embeddings_result_swapped_for_callbacks():
],
"model": "EmbeddingsGigaR",
},
request=httpx.Request(
"POST", "https://gigachat.devices.sberbank.ru/api/v1/embeddings"
),
request=httpx.Request("POST", "https://gigachat.devices.sberbank.ru/api/v1/embeddings"),
)
_, _, swapped_result = logging_obj._success_handler_helper_fn(
@ -6955,12 +6988,14 @@ def test_get_status_fields_ranks_guardrail_flagged_between_success_and_intervene
request-level guardrail_status but never mask an intervention."""
flagged = {"guardrail_status": "guardrail_flagged"}
assert _get_status_fields(
"success", [{"guardrail_status": "success"}, flagged], None
)["guardrail_status"] == "guardrail_flagged"
assert _get_status_fields(
"success", [flagged, {"guardrail_status": "guardrail_intervened"}], None
)["guardrail_status"] == "guardrail_intervened"
assert (
_get_status_fields("success", [{"guardrail_status": "success"}, flagged], None)["guardrail_status"]
== "guardrail_flagged"
)
assert (
_get_status_fields("success", [flagged, {"guardrail_status": "guardrail_intervened"}], None)["guardrail_status"]
== "guardrail_intervened"
)
def test_get_error_information_redacts_provider_key_from_upstream_url():

175
tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py
View file

@ -2,11 +2,11 @@
Unit tests for SensitiveDataMasker - List Preservation
"""
from functools import reduce
from typing import Final
import pytest
# Add the parent directory to the system path
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
@ -152,9 +152,7 @@ def test_mask_short_values_false_keeps_short_values_readable():
chars of an exception and only masks longer tails), while longer values are still
partially masked.
"""
masker = SensitiveDataMasker(
visible_prefix=50, visible_suffix=0, mask_short_values=False
)
masker = SensitiveDataMasker(visible_prefix=50, visible_suffix=0, mask_short_values=False)
short = "Test exception for structure validation"
assert masker._mask_value(short) == short
@ -202,9 +200,7 @@ def test_mask_sensitive_structure_passes_through_plain_topology_names():
from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
assert mask_sensitive_structure(["gpt-4", "claude-3-haiku"]) == ["gpt-4", "claude-3-haiku"]
assert mask_sensitive_structure([{"gpt-3.5-turbo": ["claude-3-haiku"]}]) == [
{"gpt-3.5-turbo": ["claude-3-haiku"]}
]
assert mask_sensitive_structure([{"gpt-3.5-turbo": ["claude-3-haiku"]}]) == [{"gpt-3.5-turbo": ["claude-3-haiku"]}]
assert mask_sensitive_structure(None) is None
@ -233,9 +229,7 @@ def test_mask_sensitive_structure_masks_credentials_nested_in_config_shape():
from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
secret = "sk-NESTEDINLINESECRET0987654321"
masked = mask_sensitive_structure(
[{"primary-group": [{"model": "gpt-4o", "api_key": secret}]}]
)
masked = mask_sensitive_structure([{"primary-group": [{"model": "gpt-4o", "api_key": secret}]}])
assert secret not in str(masked)
@ -282,10 +276,7 @@ def test_mask_credentials_in_payload_masks_inside_pydantic_models():
auth_dict = result["user_api_key_auth"]
assert isinstance(auth_dict, dict)
assert auth_dict["team_alias"] == "acme"
assert (
auth_dict["token"]
!= "1b01552f6e52e0d41963dd6a185bd6b074624e330999534ca7ff5adfdf622dfc"
)
assert auth_dict["token"] != "1b01552f6e52e0d41963dd6a185bd6b074624e330999534ca7ff5adfdf622dfc"
assert "*" in auth_dict["token"]
@ -314,6 +305,159 @@ def test_mask_credentials_in_payload_masks_only_sensitive_string_leaves():
assert masked.endswith(plaintext[-4:])
def _unique_dict_ids(node: object) -> frozenset[int]:
if isinstance(node, dict):
return frozenset((id(node),)).union(*(_unique_dict_ids(value) for value in node.values()))
if isinstance(node, list):
return frozenset().union(*(_unique_dict_ids(value) for value in node))
return frozenset()
def _nested_under_levels(leaf: object, levels: int) -> object:
return reduce(lambda inner, level: {f"l{level}": inner}, range(levels, 0, -1), leaf)
def test_mask_credentials_in_payload_keeps_a_shared_dict_shared():
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
shared: Final = {"api_key": "sk-shared-1234567890abcdef", "model": "gpt-4o-mini"}
result: Final = mask_credentials_in_payload({"first": shared, "second": shared})
assert result["first"] is result["second"]
assert result["first"]["model"] == "gpt-4o-mini"
assert result["first"]["api_key"] != "sk-shared-1234567890abcdef"
def test_mask_credentials_in_payload_walks_each_dag_node_once():
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
root: Final = reduce(
lambda inner, _: {"a": inner, "b": inner, "c": inner}, range(8), {"api_key": "sk-leaf-1234567890abcdef"}
)
result: Final = mask_credentials_in_payload(root)
assert len(_unique_dict_ids(root)) == 9
assert len(_unique_dict_ids(result)) == 9
assert "sk-leaf-1234567890abcdef" not in str(result)
def test_mask_credentials_in_payload_cuts_a_cycle_at_its_first_back_edge():
from litellm.litellm_core_utils.secret_redaction import REDACTED
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
node: Final[dict[str, object]] = {"api_key": "sk-cycle-1234567890abcdef"}
node["kids"] = [node] * 3
result: Final = mask_credentials_in_payload(node)
assert result["kids"] == [REDACTED, REDACTED, REDACTED]
assert result["api_key"] != "sk-cycle-1234567890abcdef"
assert len(_unique_dict_ids(result)) == 1
def test_mask_credentials_in_payload_masks_a_shared_list_only_under_a_sensitive_key():
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
shared: Final = ["sk-list-1234567890abcdef"]
plain_first: Final = mask_credentials_in_payload({"tags": shared, "api_key": shared})
assert plain_first["tags"] == ["sk-list-1234567890abcdef"]
assert plain_first["api_key"] != ["sk-list-1234567890abcdef"]
sensitive_first: Final = mask_credentials_in_payload({"api_key": shared, "tags": shared})
assert sensitive_first["api_key"] != ["sk-list-1234567890abcdef"]
assert sensitive_first["tags"] == ["sk-list-1234567890abcdef"]
def test_mask_credentials_in_payload_masks_a_shared_root_model_list_only_under_a_sensitive_key():
from pydantic import RootModel
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
shared: Final = RootModel[list[str]](["sk-root-1234567890abcdef"])
plain_first: Final = mask_credentials_in_payload({"tags": shared, "api_key": shared})
assert plain_first["tags"] == ["sk-root-1234567890abcdef"]
assert plain_first["api_key"] != ["sk-root-1234567890abcdef"]
sensitive_first: Final = mask_credentials_in_payload({"api_key": shared, "tags": shared})
assert sensitive_first["api_key"] != ["sk-root-1234567890abcdef"]
assert sensitive_first["tags"] == ["sk-root-1234567890abcdef"]
def test_mask_credentials_in_payload_masks_a_root_model_string_as_one_string():
from pydantic import RootModel
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
result: Final = mask_credentials_in_payload(
{"api_key": RootModel[str]("sk-root-1234567890abcdef"), "model": RootModel[str]("gpt-5.4-mini")}
)
assert result["model"] == "gpt-5.4-mini"
assert result["api_key"] != "sk-root-1234567890abcdef"
assert result["api_key"].startswith("sk-r")
def test_mask_credentials_in_payload_hides_containers_past_the_depth_cap():
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
from litellm.litellm_core_utils.secret_redaction import REDACTED
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
secret: Final = "sk-deep-1234567890abcdef"
cap: Final = DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
result: Final = mask_credentials_in_payload(_nested_under_levels({"api_key": secret}, cap))
assert secret not in str(result)
at_cap: Final = reduce(lambda node, level: node[f"l{level}"], range(1, cap), result)
assert at_cap == {f"l{cap}": REDACTED}
def test_mask_credentials_in_payload_treats_strings_at_the_depth_cap_per_key():
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
secret: Final = "sk-deep-1234567890abcdef"
cap: Final = DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
strings_at_cap: Final = reduce(
lambda node, level: node[f"l{level}"],
range(1, cap),
mask_credentials_in_payload(_nested_under_levels({"api_key": secret, "model": "gpt-5.4-mini"}, cap - 1)),
)
assert strings_at_cap["model"] == "gpt-5.4-mini"
assert strings_at_cap["api_key"] != secret
assert strings_at_cap["api_key"].startswith("sk-d")
def test_mask_credentials_in_payload_keeps_sibling_models_apart():
"""CPython reuses a freed temporary's id, so an id-keyed memo has to pin what it keys."""
from pydantic import BaseModel
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
class Inner(BaseModel):
label: str
api_key: str
class Outer(BaseModel):
inner: Inner
result: Final = mask_credentials_in_payload(
{
"first": Outer(inner=Inner(label="one", api_key="sk-first-1234567890abcdef")),
"second": Outer(inner=Inner(label="two", api_key="sk-second-1234567890abcdef")),
}
)
assert result["first"]["inner"]["label"] == "one"
assert result["second"]["inner"]["label"] == "two"
assert "sk-second-1234567890abcdef" not in str(result)
assert result["second"]["inner"]["api_key"].startswith("sk-s")
def test_extra_sensitive_patterns_add_to_the_defaults():
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
@ -344,6 +488,7 @@ def test_the_second_positional_argument_is_still_the_override_set():
assert masker.is_sensitive_key("session_token") is False
assert masker.is_sensitive_key("auth_token") is True
def test_redact_credentials_in_payload_leaves_no_fragment_of_the_secret():
"""A payload rendered straight to stdout cannot afford the partial reveal
mask_credentials_in_payload leaves, so every credential-named value is replaced

62
tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
View file

@ -2853,6 +2853,68 @@ def test_direct_vector_store_search_debug_log_omits_stored_credentials(caplog, i
assert "sk-embedding-s3cret" not in logged
@pytest.mark.asyncio
async def test_async_retrieve_batch_masks_presigned_auth_header_in_raw_request_log():
"""Regression: a pre-signed retrieve-batch request (Mistral, Bedrock) embeds its auth
header inside the transformed request, which pre_call logs verbatim as the raw request
body, so the provider key landed unmasked in raw_request_typed_dict and every
raw-request callback."""
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.mistral.batches.transformation import MistralBatchesConfig
provider_key = "mistral-s3cret-provider-key-123456"
job_payload = {
"id": "batch-1",
"input_files": ["file-1"],
"endpoint": "/v1/ocr",
"model": "mistral-ocr-latest",
"status": "SUCCESS",
"created_at": 1_757_400_000,
}
sent_requests = []
def _capture(request: httpx.Request) -> httpx.Response:
sent_requests.append(request)
return httpx.Response(200, json=job_payload)
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(_capture))
logging_obj = LitellmLogging(
model="mistral/mistral-ocr-latest",
messages=[],
stream=False,
call_type="batch_retrieve",
start_time=time.time(),
litellm_call_id="batch-retrieve-call-id",
function_id="batch-retrieve-function-id",
log_raw_request_response=True,
)
logging_obj.update_environment_variables(
model="mistral/mistral-ocr-latest",
optional_params={},
litellm_params={"litellm_call_id": "batch-retrieve-call-id", "metadata": {}},
)
result = await BaseLLMHTTPHandler().retrieve_batch(
batch_id="batch-1",
litellm_params={"api_key": provider_key},
provider_config=MistralBatchesConfig(),
headers={},
api_base=None,
api_key=provider_key,
logging_obj=logging_obj,
_is_async=True,
client=client,
model="mistral/mistral-ocr-latest",
)
assert result.id == "batch-1"
assert sent_requests[0].headers["Authorization"] == f"Bearer {provider_key}"
raw_request_body = logging_obj.model_call_details["raw_request_typed_dict"]["raw_request_body"]
assert provider_key not in json.dumps(raw_request_body)
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_carries_deployment_vertex_location_for_pricing(monkeypatch):
"""

0
tests/test_litellm/llms/mistral/batches/__init__.py Normal file
View file

258
tests/test_litellm/llms/mistral/batches/test_mistral_batches_transformation.py Normal file
View file

@ -0,0 +1,258 @@
"""
Regression tests for ``MistralBatchesConfig``, the BaseBatchesConfig implementation
behind ``custom_llm_provider="mistral"`` on /v1/batches.
Locks the request shape Mistral's ``POST /v1/batch/jobs`` accepts (input_files list,
model set on the job, endpoint passed through untouched so ``/v1/ocr`` batches work),
the Mistral -> OpenAI status mapping, request-count and file-id mapping, and auth.
Everything runs for real against canned httpx responses; only the API key env var is
set.
"""
import json
import httpx
import pytest
from litellm.llms.mistral.batches.transformation import MistralBatchesConfig
from litellm.llms.mistral.common_utils import MistralError
from litellm.types.llms.openai import CreateBatchRequest
from litellm.types.utils import LiteLLMBatch, LlmProviders
STATUS_MAP = {
"QUEUED": "validating",
"RUNNING": "in_progress",
"SUCCESS": "completed",
"FAILED": "failed",
"TIMEOUT_EXCEEDED": "expired",
"CANCELLATION_REQUESTED": "cancelling",
"CANCELLED": "cancelled",
}
def _job(**overrides):
base = {
"id": "8ff5e0d1-6bc2-4c3a-9f7d-0d1c2e3f4a5b",
"object": "batch",
"input_files": ["c1a2b3d4-0000-4000-8000-000000000001"],
"endpoint": "/v1/ocr",
"model": "mistral-ocr-latest",
"status": "SUCCESS",
"created_at": 1_757_400_000,
"started_at": 1_757_400_010,
"completed_at": 1_757_400_500,
"total_requests": 3,
"completed_requests": 3,
"succeeded_requests": 2,
"failed_requests": 1,
"output_file": "out-0000-4000-8000-000000000002",
"error_file": "err-0000-4000-8000-000000000003",
"errors": [],
"metadata": {"job_type": "testing"},
}
return {**base, **overrides}
def _response(payload: dict, status_code: int = 200) -> httpx.Response:
return httpx.Response(
status_code=status_code,
content=json.dumps(payload).encode(),
request=httpx.Request("GET", "https://api.mistral.ai/v1/batch/jobs/x"),
)
@pytest.fixture
def config() -> MistralBatchesConfig:
return MistralBatchesConfig()
@pytest.fixture
def api_key(monkeypatch) -> str:
monkeypatch.setenv("MISTRAL_API_KEY", "sk-mistral-test")
return "sk-mistral-test"
def test_custom_llm_provider(config):
assert config.custom_llm_provider == LlmProviders.MISTRAL
def test_create_request_maps_openai_fields_onto_mistral_job(config):
data = CreateBatchRequest(
completion_window="24h",
endpoint="/v1/ocr",
input_file_id="file-123",
metadata={"team": "docs"},
)
body = config.transform_create_batch_request(
model="mistral-ocr-latest", create_batch_data=data, optional_params={}, litellm_params={}
)
assert body == {
"input_files": ("file-123",),
"endpoint": "/v1/ocr",
"model": "mistral-ocr-latest",
"metadata": {"team": "docs"},
}
def test_create_request_omits_empty_metadata(config):
data = CreateBatchRequest(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-123",
metadata=None,
)
body = config.transform_create_batch_request(
model="mistral-small-latest", create_batch_data=data, optional_params={}, litellm_params={}
)
assert "metadata" not in body
def test_create_request_requires_input_file_and_endpoint(config):
with pytest.raises(ValueError, match="input_file_id and endpoint are required"):
config.transform_create_batch_request(
model="m",
create_batch_data=CreateBatchRequest(completion_window="24h"),
optional_params={},
litellm_params={},
)
@pytest.mark.parametrize(
"api_base,expected",
[
(None, "https://api.mistral.ai/v1/batch/jobs"),
("https://api.mistral.ai/v1", "https://api.mistral.ai/v1/batch/jobs"),
("https://proxy.example.com/", "https://proxy.example.com/v1/batch/jobs"),
],
)
def test_create_url(config, api_base, expected):
url = config.get_complete_batch_url(
api_base=api_base, api_key="k", model="m", optional_params={}, litellm_params={}, data={}
)
assert url == expected
def test_validate_environment_uses_bearer_auth(config, api_key):
headers = config.validate_environment(
headers={"x-extra": "1"}, model="m", messages=[], optional_params={}, litellm_params={}
)
assert headers == {"x-extra": "1", "Authorization": f"Bearer {api_key}"}
def test_validate_environment_explicit_key_wins(config, api_key):
headers = config.validate_environment(
headers={}, model="m", messages=[], optional_params={}, litellm_params={}, api_key="sk-explicit"
)
assert headers["Authorization"] == "Bearer sk-explicit"
def test_validate_environment_without_key_raises(config, monkeypatch):
monkeypatch.delenv("MISTRAL_API_KEY", raising=False)
with pytest.raises(ValueError, match="Missing Mistral API Key"):
config.validate_environment(headers={}, model="m", messages=[], optional_params={}, litellm_params={})
def test_create_response_maps_job_onto_openai_batch(config):
batch = config.transform_create_batch_response(
model="mistral-ocr-latest",
raw_response=_response(_job(status="QUEUED", started_at=None, completed_at=None)),
logging_obj=None,
litellm_params={},
)
assert isinstance(batch, LiteLLMBatch)
assert batch.id == "8ff5e0d1-6bc2-4c3a-9f7d-0d1c2e3f4a5b"
assert batch.endpoint == "/v1/ocr"
assert batch.input_file_id == "c1a2b3d4-0000-4000-8000-000000000001"
assert batch.status == "validating"
assert batch.created_at == 1_757_400_000
assert batch.in_progress_at is None
assert batch.completed_at is None
assert batch.metadata == {"job_type": "testing"}
def test_retrieve_request_is_presigned_get_with_auth(config, api_key):
req = config.transform_retrieve_batch_request(
batch_id="job/with slash", optional_params={}, litellm_params={"api_base": "https://api.mistral.ai"}
)
assert req["method"] == "GET"
assert req["url"] == "https://api.mistral.ai/v1/batch/jobs/job%2Fwith%20slash"
assert req["headers"] == {"Authorization": f"Bearer {api_key}"}
def test_retrieve_request_prefers_litellm_params_api_key(config, api_key):
req = config.transform_retrieve_batch_request(
batch_id="job-1", optional_params={}, litellm_params={"api_key": "sk-from-deployment"}
)
assert req["headers"]["Authorization"] == "Bearer sk-from-deployment"
@pytest.mark.parametrize("mistral_status,openai_status", sorted(STATUS_MAP.items()))
def test_retrieve_response_status_mapping(config, mistral_status, openai_status):
batch = config.transform_retrieve_batch_response(
model=None, raw_response=_response(_job(status=mistral_status)), logging_obj=None, litellm_params={}
)
assert batch.status == openai_status
@pytest.mark.parametrize(
"mistral_status,populated_field",
[
("SUCCESS", "completed_at"),
("FAILED", "failed_at"),
("TIMEOUT_EXCEEDED", "expired_at"),
("CANCELLED", "cancelled_at"),
],
)
def test_retrieve_response_terminal_timestamp_lands_on_matching_field(config, mistral_status, populated_field):
batch = config.transform_retrieve_batch_response(
model=None, raw_response=_response(_job(status=mistral_status)), logging_obj=None, litellm_params={}
)
terminal_fields = {"completed_at", "failed_at", "expired_at", "cancelled_at"}
assert getattr(batch, populated_field) == 1_757_400_500
for other in terminal_fields - {populated_field}:
assert getattr(batch, other) is None
assert batch.in_progress_at == 1_757_400_010
def test_retrieve_response_maps_counts_and_files(config):
batch = config.transform_retrieve_batch_response(
model=None, raw_response=_response(_job()), logging_obj=None, litellm_params={}
)
assert batch.request_counts.total == 3
assert batch.request_counts.completed == 2
assert batch.request_counts.failed == 1
assert batch.output_file_id == "out-0000-4000-8000-000000000002"
assert batch.error_file_id == "err-0000-4000-8000-000000000003"
assert batch.errors is None
def test_retrieve_response_surfaces_job_errors(config):
batch = config.transform_retrieve_batch_response(
model=None,
raw_response=_response(
_job(status="FAILED", errors=[{"message": "invalid document", "count": 2}, {"message": "timeout"}])
),
logging_obj=None,
litellm_params={},
)
assert [e.message for e in batch.errors.data] == ["invalid document (x2)", "timeout"]
def test_retrieve_response_without_files_or_input(config):
batch = config.transform_retrieve_batch_response(
model=None,
raw_response=_response(_job(input_files=[], output_file=None, error_file=None, metadata=None)),
logging_obj=None,
litellm_params={},
)
assert batch.input_file_id == ""
assert batch.output_file_id is None
assert batch.error_file_id is None
assert batch.metadata is None
def test_get_error_class(config):
err = config.get_error_class("nope", 401, {"x-request-id": "r1"})
assert isinstance(err, MistralError)
assert err.status_code == 401
assert err.message == "nope"

0
tests/test_litellm/llms/mistral/files/__init__.py Normal file
View file

249
tests/test_litellm/llms/mistral/files/test_mistral_files_transformation.py Normal file
View file

@ -0,0 +1,249 @@
"""
Regression tests for ``MistralFilesConfig``, the BaseFilesConfig implementation behind
``custom_llm_provider="mistral"`` on /v1/files.
Locks the URL routing for each file operation, the multipart upload shape Mistral's
``POST /v1/files`` accepts (purpose restricted to fine-tune/batch/ocr), and the
Mistral -> OpenAI file object mapping. Runs against canned httpx responses.
"""
import json
import httpx
import pytest
from openai.types.file_deleted import FileDeleted
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.mistral.files.transformation import MistralFilesConfig
from litellm.types.llms.openai import CreateFileRequest, FileContentRequest, OpenAIFileObject
from litellm.types.utils import LlmProviders
FILE_ID = "497f6eca-6276-4993-bfeb-53cbbbba6f09"
def _file(**overrides):
base = {
"id": FILE_ID,
"object": "file",
"bytes": 13000,
"created_at": 1_716_963_433,
"filename": "batch_input.jsonl",
"purpose": "batch",
"sample_type": "batch_request",
"num_lines": 3,
"source": "upload",
}
return {**base, **overrides}
def _response(payload) -> httpx.Response:
return httpx.Response(
status_code=200,
content=json.dumps(payload).encode(),
request=httpx.Request("GET", "https://api.mistral.ai/v1/files"),
)
@pytest.fixture
def config() -> MistralFilesConfig:
return MistralFilesConfig()
@pytest.fixture
def api_key(monkeypatch) -> str:
monkeypatch.setenv("MISTRAL_API_KEY", "sk-mistral-test")
return "sk-mistral-test"
def test_custom_llm_provider(config):
assert config.custom_llm_provider == LlmProviders.MISTRAL
@pytest.mark.parametrize(
"api_base,expected",
[
(None, "https://api.mistral.ai/v1/files"),
("https://api.mistral.ai/v1/", "https://api.mistral.ai/v1/files"),
("https://proxy.example.com", "https://proxy.example.com/v1/files"),
],
)
def test_upload_url(config, api_base, expected):
url = config.get_complete_url(api_base=api_base, api_key="k", model="", optional_params={}, litellm_params={})
assert url == expected
def test_validate_environment_uses_bearer_auth(config, api_key):
headers = config.validate_environment(headers={}, model="", messages=[], optional_params={}, litellm_params={})
assert headers == {"Authorization": f"Bearer {api_key}"}
def test_upload_request_is_multipart_with_batch_purpose(config):
body = config.transform_create_file_request(
model="",
create_file_data=CreateFileRequest(
file=("in.jsonl", b'{"custom_id":"0"}\n', "application/jsonl"), purpose="batch"
),
optional_params={},
litellm_params={},
)
assert body == {
"file": ("in.jsonl", b'{"custom_id":"0"}\n', "application/jsonl"),
"purpose": (None, "batch"),
}
@pytest.mark.parametrize("purpose", ["batch", "fine-tune", "ocr"])
def test_upload_request_passes_mistral_purposes_through(config, purpose):
body = config.transform_create_file_request(
model="",
create_file_data=CreateFileRequest(file=("f.bin", b"x"), purpose=purpose),
optional_params={},
litellm_params={},
)
assert body["purpose"] == (None, purpose)
def test_upload_request_maps_user_data_onto_ocr(config):
body = config.transform_create_file_request(
model="",
create_file_data=CreateFileRequest(file=("scan.pdf", b"%PDF"), purpose="user_data"),
optional_params={},
litellm_params={},
)
assert body["purpose"] == (None, "ocr")
@pytest.mark.parametrize("purpose", ["assistants", "vision", "evals"])
def test_upload_request_rejects_purposes_mistral_lacks(config, purpose):
"""Regression: these used to be silently rewritten to ``batch``, so an upload that skipped the
proxy's batch-only validation and guardrails still landed on Mistral as a batch input file. The
rejection is a 400 provider error, so the proxy answers invalid_request_error instead of a 500."""
with pytest.raises(BaseLLMException, match=f"purpose={purpose!r}") as exc_info:
config.transform_create_file_request(
model="",
create_file_data=CreateFileRequest(file=("f.bin", b"x"), purpose=purpose),
optional_params={},
litellm_params={},
)
assert exc_info.value.status_code == 400
def test_upload_request_requires_file(config):
with pytest.raises(ValueError, match="File data is required"):
config.transform_create_file_request(
model="", create_file_data=CreateFileRequest(purpose="batch"), optional_params={}, litellm_params={}
)
def test_upload_response_maps_onto_openai_file_object(config):
obj = config.transform_create_file_response(
model=None, raw_response=_response(_file()), logging_obj=None, litellm_params={}
)
assert obj == OpenAIFileObject(
id=FILE_ID,
bytes=13000,
created_at=1_716_963_433,
filename="batch_input.jsonl",
object="file",
purpose="batch",
status="uploaded",
)
def test_file_response_with_ocr_purpose_maps_onto_user_data(config):
obj = config.transform_retrieve_file_response(
raw_response=_response(_file(purpose="ocr", expires_at=1_800_000_000)), logging_obj=None, litellm_params={}
)
assert obj.purpose == "user_data"
assert obj.expires_at == 1_800_000_000
@pytest.mark.parametrize("purpose", ["playground", "audio", "code_interpreter"])
def test_files_with_purposes_mistral_never_lets_us_upload_still_read_back(config, purpose):
"""Regression: Mistral's live API returns purposes its upload endpoint rejects for files
other Mistral products created, and both the unfiltered list and a retrieve of such a file
used to fail validation, so one playground file 500'd ``GET /v1/files`` for the whole key."""
retrieved = config.transform_retrieve_file_response(
raw_response=_response(_file(purpose=purpose)), logging_obj=None, litellm_params={}
)
assert retrieved.purpose == "user_data"
listed = config.transform_list_files_response(
raw_response=_response({"data": [_file(purpose=purpose), _file(id="second")], "object": "list", "total": 2}),
logging_obj=None,
litellm_params={},
)
assert [(f.id, f.purpose) for f in listed] == [(FILE_ID, "user_data"), ("second", "batch")]
@pytest.mark.parametrize(
"method,suffix",
[
("transform_retrieve_file_request", ""),
("transform_delete_file_request", ""),
],
)
def test_single_file_urls_encode_id_and_honor_api_base(config, method, suffix):
url, params = getattr(config, method)(
file_id="id/with slash", optional_params={}, litellm_params={"api_base": "https://mistral.internal/v1"}
)
assert url == f"https://mistral.internal/v1/files/id%2Fwith%20slash{suffix}"
assert params == {}
def test_file_content_url(config):
url, params = config.transform_file_content_request(
file_content_request=FileContentRequest(file_id=FILE_ID), optional_params={}, litellm_params={}
)
assert url == f"https://api.mistral.ai/v1/files/{FILE_ID}/content"
assert params == {}
def test_file_content_response_is_binary_passthrough(config):
raw = httpx.Response(
200, content=b'{"custom_id":"0","response":{"status_code":200}}\n', request=httpx.Request("GET", "https://x")
)
out = config.transform_file_content_response(raw_response=raw, logging_obj=None, litellm_params={})
assert out.content == b'{"custom_id":"0","response":{"status_code":200}}\n'
def test_delete_response(config):
out = config.transform_delete_file_response(
raw_response=_response({"id": FILE_ID, "object": "file", "deleted": True}), logging_obj=None, litellm_params={}
)
assert out == FileDeleted(id=FILE_ID, deleted=True, object="file")
def test_list_request_filters_by_mapped_purpose(config):
url, params = config.transform_list_files_request(purpose="batch", optional_params={}, litellm_params={})
assert url == "https://api.mistral.ai/v1/files"
assert params == {"purpose": "batch"}
_, no_params = config.transform_list_files_request(purpose=None, optional_params={}, litellm_params={})
assert no_params == {}
def test_list_request_accepts_the_purpose_an_ocr_file_reads_back_as(config):
"""Regression: an OCR file reads back as ``purpose=user_data``, and listing with that purpose
used to raise, so ``files.list(purpose=file.purpose)`` could never find OCR files."""
ocr_file = config.transform_retrieve_file_response(
raw_response=_response(_file(purpose="ocr")), logging_obj=None, litellm_params={}
)
_, params = config.transform_list_files_request(purpose=ocr_file.purpose, optional_params={}, litellm_params={})
assert params == {"purpose": "ocr"}
def test_list_request_rejects_purposes_mistral_lacks(config):
with pytest.raises(BaseLLMException, match="purpose='assistants'") as exc_info:
config.transform_list_files_request(purpose="assistants", optional_params={}, litellm_params={})
assert exc_info.value.status_code == 400
def test_list_response(config):
out = config.transform_list_files_response(
raw_response=_response(
{"data": [_file(), _file(id="second", filename="b.jsonl")], "object": "list", "total": 2}
),
logging_obj=None,
litellm_params={},
)
assert [f.id for f in out] == [FILE_ID, "second"]
assert out[1].filename == "b.jsonl"

15
tests/test_litellm/proxy/auth/test_model_checks.py
View file

@ -879,3 +879,18 @@ def test_get_complete_model_list_sentinel_only_grants_nothing():
infer_model_from_keys=False,
)
assert result == []
def test_transcribe_is_a_known_provider_for_wildcard_expansion():
import litellm
from litellm.proxy.auth.model_checks import (
get_known_models_from_wildcard,
get_provider_models,
)
assert "transcribe" in litellm.models_by_provider
assert "transcribe/StartTranscriptionJob" in litellm.models_by_provider["transcribe"]
assert get_provider_models("transcribe") == ["transcribe/StartTranscriptionJob"]
assert get_known_models_from_wildcard("transcribe/*") == [
"transcribe/StartTranscriptionJob"
]

157
tests/test_litellm/proxy/batches_endpoints/test_endpoints.py
View file

@ -51,6 +51,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
from litellm.proxy.utils import ProxyLogging
from litellm.router import Router
from litellm.types.llms.openai import BatchJobStatus
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
from litellm.types.utils import CredentialItem, LiteLLMBatch
from fastapi import Request, Response
@ -178,6 +179,10 @@ def harness():
logging.get_proxy_hook = MagicMock(return_value=None)
router = MagicMock(spec=Router)
router.model_group_alias = {}
router.get_model_access_groups = MagicMock(return_value={})
router.resolve_model_name_from_model_id = MagicMock(side_effect=lambda model_id: model_id)
router.model_list = []
router.acreate_batch = AsyncMock(return_value=make_batch())
router.get_deployment_credentials_with_provider = MagicMock(side_effect=_creds_lookup)
@ -1184,8 +1189,13 @@ def retrieve_harness():
logging.get_proxy_hook = MagicMock(return_value=None)
router = MagicMock(spec=Router)
router.model_group_alias = {}
router.get_model_access_groups = MagicMock(return_value={})
router.resolve_model_name_from_model_id = MagicMock(side_effect=lambda model_id: model_id)
router.model_list = []
router.aretrieve_batch = AsyncMock(return_value=make_batch())
router.get_deployment_credentials_with_provider = MagicMock(side_effect=_creds_lookup)
router.get_credential_deployment = MagicMock(return_value=None)
pre_call = AsyncMock(side_effect=lambda **kw: (data_holder["data"], MagicMock()))
get_headers = MagicMock(return_value={})
@ -1304,6 +1314,30 @@ async def test_retrieve__model_encoded_id(retrieve_harness):
assert retrieve_harness.update_batch_in_db.call_args.kwargs["operation"] == "retrieve"
@pytest.mark.asyncio
async def test_retrieve__model_encoded_id__stamps_deployment_model_info_for_cost(retrieve_harness):
"""Regression: this path calls litellm.aretrieve_batch directly, so nothing stamped the
deployment's model_info the way the router does for routed calls. Cost tracking then never
saw the deployment id, and a completed batch on a deployment with its own per-page pricing
was billed at the published rate with an empty model_id on the spend row."""
retrieve_harness.router.get_credential_deployment.return_value = Deployment(
model_name="azure-gpt",
litellm_params=LiteLLM_Params(model="azure/gpt-4o"),
model_info=ModelInfo(id="dep-123"),
)
retrieve_harness.pre_call.side_effect = lambda **kw: (
{**retrieve_harness.data["data"], "litellm_metadata": {"user_api_key_alias": "qa-key"}},
MagicMock(),
)
await call_retrieve(retrieve_harness, AZURE_BATCH_ID)
retrieve_harness.router.get_credential_deployment.assert_called_once_with(model_id="azure/gpt-4o")
litellm_metadata = retrieve_harness.aretrieve_kwargs()["litellm_metadata"]
assert litellm_metadata["model_info"]["id"] == "dep-123"
assert litellm_metadata["user_api_key_alias"] == "qa-key"
@pytest.mark.asyncio
async def test_retrieve__model_encoded_id__forwards_decoded_model_not_deployment(
retrieve_harness,
@ -1639,6 +1673,10 @@ def list_harness():
logging.get_proxy_hook = MagicMock(return_value=None)
router = MagicMock(spec=Router)
router.model_group_alias = {}
router.get_model_access_groups = MagicMock(return_value={})
router.resolve_model_name_from_model_id = MagicMock(side_effect=lambda model_id: model_id)
router.model_list = []
router.alist_batches = AsyncMock(return_value=FakeListPage([]))
router.get_deployment_credentials_with_provider = MagicMock(side_effect=_creds_lookup)
@ -2053,6 +2091,10 @@ def cancel_harness():
logging.get_proxy_hook = MagicMock(return_value=None)
router = MagicMock(spec=Router)
router.model_group_alias = {}
router.get_model_access_groups = MagicMock(return_value={})
router.resolve_model_name_from_model_id = MagicMock(side_effect=lambda model_id: model_id)
router.model_list = []
router.acancel_batch = AsyncMock(return_value=make_batch())
router.get_deployment_credentials_with_provider = MagicMock(side_effect=_creds_lookup)
@ -2774,8 +2816,6 @@ async def test_cancel__unified_batch_id_allowed_when_managed_files_required(canc
assert cancel_harness.router_acancel.call_count == 1
@pytest.mark.asyncio
async def test_retrieve__managed_batch_defers_cost_to_the_poller_when_it_is_running(retrieve_harness):
with patch.object(endpoints, "batch_cost_poller_is_active", MagicMock(return_value=True)):
@ -2803,3 +2843,116 @@ async def test_retrieve__raw_batch_id_is_untouched_by_the_poller_handoff(retriev
metadata = retrieve_harness.litellm_aretrieve.await_args.kwargs.get("litellm_metadata") or {}
assert metadata.get("batch_ignore_default_logging") is None
def _key_restricted_to(*models: str) -> UserAPIKeyAuth:
return UserAPIKeyAuth(api_key="sk-restricted", team_id="team-a", team_models=list(models), models=list(models))
@pytest.mark.asyncio
async def test_create__header_model_rejects_key_without_model_grant(harness):
"""A key not granted the model named in x-litellm-model must not receive that deployment's credentials."""
set_body(harness, {"input_file_id": "file-plain", "endpoint": "/v1/chat/completions", "completion_window": "24h"})
with pytest.raises(ProxyException) as exc_info:
await call_create(harness, user=_key_restricted_to("azure/gpt-4o"), headers={"x-litellm-model": "vertex-model"})
assert exc_info.value.code == "403"
harness.creds_resolver.assert_not_called()
harness.litellm_acreate.assert_not_called()
@pytest.mark.asyncio
async def test_create__header_model_allows_key_with_model_grant(harness):
set_body(harness, {"input_file_id": "file-plain", "endpoint": "/v1/chat/completions", "completion_window": "24h"})
await call_create(harness, user=_key_restricted_to("vertex-model"), headers={"x-litellm-model": "vertex-model"})
harness.creds_resolver.assert_called_once_with(model_id="vertex-model")
assert harness.acreate_kwargs()["custom_llm_provider"] == "vertex_ai"
@pytest.mark.asyncio
async def test_retrieve__model_encoded_id_rejects_key_without_model_grant(retrieve_harness):
"""The model embedded in a batch id is caller-controlled, so it is checked against the key's grants too."""
with pytest.raises(ProxyException) as exc_info:
await call_retrieve(retrieve_harness, AZURE_BATCH_ID, user=_key_restricted_to("vertex-model"))
assert exc_info.value.code == "403"
retrieve_harness.creds_resolver.assert_not_called()
retrieve_harness.litellm_aretrieve.assert_not_called()
@pytest.mark.asyncio
async def test_cancel__model_encoded_id_rejects_key_without_model_grant(cancel_harness):
with pytest.raises(ProxyException) as exc_info:
await call_cancel(cancel_harness, AZURE_BATCH_ID, user=_key_restricted_to("vertex-model"))
assert exc_info.value.code == "403"
cancel_harness.creds_resolver.assert_not_called()
cancel_harness.litellm_acancel.assert_not_called()
def _b64_unified_id(decoded: str) -> str:
return base64.urlsafe_b64encode(decoded.encode()).decode().rstrip("=")
UNIFIED_FILE_ID_FOR_GPT4O_MINI = _b64_unified_id(
"litellm_proxy:application/octet-stream;unified_id,c4843482-b176-4901-8292-7523fd0f2c6e;"
"target_model_names,gpt-4o-mini;llm_output_file_id,file-provider;llm_output_file_model_id,dep-1"
)
UNIFIED_BATCH_ID_FOR_GPT4O_MINI = _b64_unified_id(UNIFIED_BATCH_ID)
@pytest.mark.asyncio
async def test_create__unified_file_id_rejects_key_without_model_grant(harness):
"""The model carried inside a unified file id is caller-controlled too, so it is checked against the key's grants."""
set_body(
harness,
{
"input_file_id": UNIFIED_FILE_ID_FOR_GPT4O_MINI,
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
},
)
with pytest.raises(ProxyException) as exc_info:
await call_create(harness, user=_key_restricted_to("vertex-model"))
assert exc_info.value.code == "403"
harness.router_acreate.assert_not_called()
harness.litellm_acreate.assert_not_called()
@pytest.mark.asyncio
async def test_retrieve__unified_batch_id_rejects_key_without_model_grant(retrieve_harness):
with pytest.raises(ProxyException) as exc_info:
await call_retrieve(retrieve_harness, UNIFIED_BATCH_ID_FOR_GPT4O_MINI, user=_key_restricted_to("vertex-model"))
assert exc_info.value.code == "403"
retrieve_harness.router_aretrieve.assert_not_called()
retrieve_harness.creds_resolver.assert_not_called()
@pytest.mark.asyncio
async def test_retrieve__unified_batch_id_rejects_key_without_model_grant_before_db_terminal_shortcut(
retrieve_harness,
):
retrieve_harness.get_batch_from_db.return_value = (MagicMock(), make_batch(id="batch-from-db", status="completed"))
with pytest.raises(ProxyException) as exc_info:
await call_retrieve(retrieve_harness, UNIFIED_BATCH_ID_FOR_GPT4O_MINI, user=_key_restricted_to("vertex-model"))
assert exc_info.value.code == "403"
retrieve_harness.logging.post_call_success_hook.assert_not_called()
retrieve_harness.ensure_managed_files.assert_not_called()
retrieve_harness.router_aretrieve.assert_not_called()
@pytest.mark.asyncio
async def test_cancel__unified_batch_id_rejects_key_without_model_grant(cancel_harness):
with pytest.raises(ProxyException) as exc_info:
await call_cancel(cancel_harness, UNIFIED_BATCH_ID_FOR_GPT4O_MINI, user=_key_restricted_to("vertex-model"))
assert exc_info.value.code == "403"
cancel_harness.router_acancel.assert_not_called()

49
tests/test_litellm/proxy/guardrails/guardrail_hooks/azure/test_azure_prompt_shield.py
View file

@ -7,6 +7,7 @@ from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_hooks.azure.prompt_shield import (
AzureContentSafetyPromptShieldGuardrail,
)
from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler
from litellm.types.guardrails import LitellmParams
@ -635,3 +636,51 @@ def test_update_in_memory_litellm_params_dead_env_credential_rejected_untouched(
assert guardrail.api_key == "azure_prompt_shield_api_key"
assert guardrail.price_per_1000_text_records == 0.38
@pytest.mark.asyncio
async def test_config_without_api_version_calls_documented_azure_api_version():
handler = InMemoryGuardrailHandler()
registered = handler.initialize_guardrail(
guardrail={
"guardrail_name": "azure-prompt-shield-no-api-version",
"litellm_params": {
"guardrail": "azure/prompt_shield",
"mode": "pre_call",
"api_key": "azure_prompt_shield_api_key",
"api_base": "https://example.cognitiveservices.azure.com",
},
}
)
assert registered is not None
guardrail = handler.guardrail_id_to_custom_guardrail[registered["guardrail_id"]]
assert isinstance(guardrail, AzureContentSafetyPromptShieldGuardrail)
with patch.object(guardrail.async_handler, "post", return_value=_shield_response(False)) as mock_post:
result = await guardrail.apply_guardrail(inputs={"texts": ["hello"]}, request_data={}, input_type="request")
assert result == {"texts": ["hello"]}
assert mock_post.call_args.kwargs["url"] == (
"https://example.cognitiveservices.azure.com/contentsafety/text:shieldPrompt?api-version=2024-09-01"
)
@pytest.mark.asyncio
async def test_update_without_api_version_keeps_documented_azure_api_version():
guardrail = _shield_guardrail()
guardrail.update_in_memory_litellm_params(
LitellmParams(
guardrail="azure/prompt_shield",
mode="pre_call",
api_key="azure_prompt_shield_api_key",
api_base="https://example.cognitiveservices.azure.com",
)
)
with patch.object(guardrail.async_handler, "post", return_value=_shield_response(False)) as mock_post:
result = await guardrail.apply_guardrail(inputs={"texts": ["hello"]}, request_data={}, input_type="request")
assert result == {"texts": ["hello"]}
assert mock_post.call_args.kwargs["url"] == (
"https://example.cognitiveservices.azure.com/contentsafety/text:shieldPrompt?api-version=2024-09-01"
)

59
tests/test_litellm/proxy/guardrails/guardrail_hooks/azure/test_azure_text_moderation.py
View file

@ -4,6 +4,7 @@ import pytest
from fastapi import HTTPException
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler
from litellm.proxy.guardrails.guardrail_hooks.azure.text_moderation import (
AzureContentSafetyTextModerationGuardrail,
)
@ -463,3 +464,61 @@ async def test_apply_guardrail_handles_missing_texts_key():
mock_post.assert_not_called()
assert result == {"images": ["x"]}
@pytest.mark.asyncio
async def test_config_without_api_version_calls_documented_azure_api_version():
handler = InMemoryGuardrailHandler()
registered = handler.initialize_guardrail(
guardrail={
"guardrail_name": "azure-text-moderation-no-api-version",
"litellm_params": {
"guardrail": "azure/text_moderations",
"mode": "pre_call",
"api_key": "azure_text_moderation_api_key",
"api_base": "https://example.cognitiveservices.azure.com",
},
}
)
assert registered is not None
guardrail = handler.guardrail_id_to_custom_guardrail[registered["guardrail_id"]]
assert isinstance(guardrail, AzureContentSafetyTextModerationGuardrail)
with patch.object(guardrail.async_handler, "post", return_value=_moderation_response(0)) as mock_post:
result = await guardrail.apply_guardrail(inputs={"texts": ["hello"]}, request_data={}, input_type="request")
assert result == {"texts": ["hello"]}
assert mock_post.call_args.kwargs["url"] == (
"https://example.cognitiveservices.azure.com/contentsafety/text:analyze?api-version=2024-09-01"
)
@pytest.mark.parametrize(
("stored_api_version", "expected_api_version"),
[("v1", "2024-09-01"), ("2023-10-01", "2023-10-01")],
)
@pytest.mark.asyncio
async def test_guardrail_loaded_with_stored_api_version_calls_azure_at(stored_api_version, expected_api_version):
handler = InMemoryGuardrailHandler()
registered = handler.initialize_guardrail(
guardrail={
"guardrail_name": f"azure-text-moderation-stored-{stored_api_version}",
"litellm_params": {
"guardrail": "azure/text_moderations",
"mode": "pre_call",
"api_key": "azure_text_moderation_api_key",
"api_base": "https://example.cognitiveservices.azure.com",
"api_version": stored_api_version,
},
}
)
assert registered is not None
guardrail = handler.guardrail_id_to_custom_guardrail[registered["guardrail_id"]]
with patch.object(guardrail.async_handler, "post", return_value=_moderation_response(0)) as mock_post:
result = await guardrail.apply_guardrail(inputs={"texts": ["hello"]}, request_data={}, input_type="request")
assert result == {"texts": ["hello"]}
assert mock_post.call_args.kwargs["url"] == (
f"https://example.cognitiveservices.azure.com/contentsafety/text:analyze?api-version={expected_api_version}"
)

39
tests/test_litellm/proxy/guardrails/guardrail_hooks/test_javelin.py Normal file
View file

@ -0,0 +1,39 @@
from unittest.mock import Mock, patch
import pytest
from litellm.proxy.guardrails.guardrail_hooks.javelin.javelin import JavelinGuardrail
from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler
from litellm.types.guardrails import GuardrailEventHooks
@pytest.mark.asyncio
async def test_config_without_api_version_calls_javelin_v1():
handler = InMemoryGuardrailHandler()
registered = handler.initialize_guardrail(
guardrail={
"guardrail_name": "javelin-no-api-version",
"litellm_params": {
"guardrail": "javelin",
"mode": "pre_call",
"api_key": "javelin_api_key",
"api_base": "https://javelin.example",
"guard_name": "trustsafety",
},
}
)
assert registered is not None
guardrail = handler.guardrail_id_to_custom_guardrail[registered["guardrail_id"]]
assert isinstance(guardrail, JavelinGuardrail)
assessments = [{"trustsafety": {"request_reject": False}}]
response = Mock()
response.json.return_value = {"assessments": assessments}
with patch.object(guardrail.async_handler, "post", return_value=response) as mock_post:
result = await guardrail.call_javelin_guard(
request={"input": {"text": "hello"}, "config": None, "metadata": None},
event_type=GuardrailEventHooks.pre_call,
)
assert result == {"assessments": assessments}
assert mock_post.call_args.kwargs["url"] == "https://javelin.example/v1/guardrail/trustsafety/apply"

78
tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py
View file

@ -1,11 +1,13 @@
import asyncio
import json
import logging
from datetime import datetime
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY
from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth
from litellm.proxy.collector import SpendEventConsumer
@ -2540,3 +2542,79 @@ async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_
== "/chat/completions: Invalid model name passed in. Call `/v1/models` to view available models for your key."
)
assert error_information["error_class"] == "ProxyModelNotFoundError"
class _NeverStringifiedMetadataValue:
def __repr__(self) -> str:
raise AssertionError("a request metadata value was stringified by the cost tracking failure path")
__str__ = __repr__
def _spend_write_kwargs_with_metadata_value(metadata_value: object) -> dict:
return {
"call_type": "acompletion",
"model": "gpt-5.4-mini",
"litellm_call_id": "test-call-id",
"stream": False,
"response_cost": 4.725e-05,
"litellm_params": {
"metadata": {
"user_api_key": "hashed-key",
"user_api_key_user_id": "user-1",
"user_context": metadata_value,
"headers": {"user-agent": metadata_value},
},
},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("log_level", [logging.WARNING, logging.DEBUG])
async def test_track_cost_callback_failure_alert_never_carries_request_metadata_values(log_level):
logger: Final = _ProxyDBLogger()
records: list[logging.LogRecord] = []
handler: Final = logging.Handler()
handler.emit = records.append
previous_level: Final = verbose_proxy_logger.level
verbose_proxy_logger.setLevel(log_level)
verbose_proxy_logger.addHandler(handler)
try:
with patch( # test-quality-ok: callback imports proxy_logging_obj off proxy_server in its body, no seam
"litellm.proxy.proxy_server.proxy_logging_obj"
) as mock_proxy_logging:
mock_proxy_logging.failed_tracking_alert = AsyncMock()
mock_proxy_logging.db_spend_update_writer = MagicMock()
mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(
side_effect=Exception("READONLY You can't write against a read only replica.")
)
await logger._PROXY_track_cost_callback(
kwargs=_spend_write_kwargs_with_metadata_value(_NeverStringifiedMetadataValue()),
completion_response=ModelResponse(),
start_time=datetime.now(),
end_time=datetime.now(),
)
await asyncio.sleep(0)
finally:
verbose_proxy_logger.removeHandler(handler)
verbose_proxy_logger.setLevel(previous_level)
mock_proxy_logging.failed_tracking_alert.assert_awaited_once()
alert: Final = mock_proxy_logging.failed_tracking_alert.await_args.kwargs
assert alert["failing_model"] == "gpt-5.4-mini"
assert "READONLY You can't write against a read only replica." in alert["error_message"]
assert "model: gpt-5.4-mini" in alert["error_message"]
assert "call_type: acompletion" in alert["error_message"]
failure_debug_lines: Final = [
record.getMessage()
for record in records
if record.levelno == logging.DEBUG and "Cost tracking callback failed" in record.getMessage()
]
if log_level == logging.DEBUG:
assert len(failure_debug_lines) == 1
assert "user_context" in failure_debug_lines[0]
assert "headers" in failure_debug_lines[0]
else:
assert failure_debug_lines == []

243
tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py
View file

@ -1877,7 +1877,7 @@ def test_get_file_content_streams_openai_direct_path(
monkeypatch.setattr(litellm, "afile_content", _mock_afile_content)
monkeypatch.setattr(
"litellm.proxy.openai_files_endpoints.files_endpoints.handle_model_based_routing",
lambda **kwargs: (False, None, None, None),
AsyncMock(return_value=(False, None, None, None)),
)
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
@ -1942,15 +1942,17 @@ def test_get_file_content_routed_provider_skips_streaming_when_resolved_provider
)
monkeypatch.setattr(
"litellm.proxy.openai_files_endpoints.files_endpoints.handle_model_based_routing",
lambda **kwargs: (
True,
"azure-gpt-3-5-turbo",
"file-original-123",
{
"custom_llm_provider": "azure",
"api_key": "azure-key",
"api_base": "https://azure.example.com",
},
AsyncMock(
return_value=(
True,
"azure-gpt-3-5-turbo",
"file-original-123",
{
"custom_llm_provider": "azure",
"api_key": "azure-key",
"api_base": "https://azure.example.com",
},
)
),
)
@ -2015,7 +2017,7 @@ def test_get_file_content_non_openai_provider_skips_streaming_handler(
)
monkeypatch.setattr(
"litellm.proxy.openai_files_endpoints.files_endpoints.handle_model_based_routing",
lambda **kwargs: (False, None, None, None),
AsyncMock(return_value=(False, None, None, None)),
)
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
@ -2520,14 +2522,16 @@ def test_list_files_model_routing_does_not_forward_custom_llm_provider_twice(
monkeypatch.setattr(litellm, "afile_list", _mock_afile_list)
monkeypatch.setattr(
"litellm.proxy.openai_files_endpoints.files_endpoints.handle_model_based_routing",
lambda **kwargs: (
True,
"azure-gpt-4o",
None,
{
"custom_llm_provider": "azure",
"api_key": "azure-key",
},
AsyncMock(
return_value=(
True,
"azure-gpt-4o",
None,
{
"custom_llm_provider": "azure",
"api_key": "azure-key",
},
)
),
)
@ -5224,3 +5228,204 @@ def test_get_file_content_keeps_the_status_of_a_rejection_raised_inside_the_rout
error = response.json()["error"]
assert error["message"].startswith("Storage backend error")
assert (error["type"], error["param"], error["code"]) == ("invalid_request_error", "file_id", "400")
def test_get_file_model_routed_id_forwards_deployment_provider(mocker: MockerFixture, monkeypatch):
"""
Regression: a file id encoded with a non-OpenAI deployment (here Mistral) must be
retrieved from that deployment's provider. Before the fix the retrieve path only
forwarded the credentials and let ``custom_llm_provider`` default to openai, so a
Mistral file id was sent to api.openai.com with the Mistral key and 401'd.
"""
import litellm.proxy.proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
from litellm.proxy.openai_files_endpoints.common_utils import encode_file_id_with_model
router = Router(
model_list=[
{
"model_name": "mistral-ocr",
"litellm_params": {"model": "mistral/mistral-ocr-latest", "api_key": "mistral-key"},
"model_info": {"id": "mistral-ocr-id"},
}
]
)
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router)
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router)
proxy_logging_obj.update_request_status = mocker.AsyncMock()
proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock()
captured_kwargs: dict = {}
async def _mock_afile_retrieve(**kwargs):
captured_kwargs.update(kwargs)
return OpenAIFileObject(
id="7a13fa8e-fcf8-42c5-aa61-c93c10e2c7df",
object="file",
bytes=2,
created_at=1234567890,
filename="batch.jsonl",
purpose="batch",
status="uploaded",
)
monkeypatch.setattr(litellm, "afile_retrieve", _mock_afile_retrieve)
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
api_key="test-key", user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test-user"
)
encoded_id = encode_file_id_with_model("7a13fa8e-fcf8-42c5-aa61-c93c10e2c7df", "mistral-ocr")
try:
response = client.get(f"/v1/files/{encoded_id}", headers={"Authorization": "Bearer test-key"})
finally:
app.dependency_overrides.pop(ps.user_api_key_auth, None)
assert response.status_code == 200, response.text
assert captured_kwargs["custom_llm_provider"] == "mistral"
assert captured_kwargs["api_key"] == "mistral-key"
assert captured_kwargs["file_id"] == "7a13fa8e-fcf8-42c5-aa61-c93c10e2c7df"
assert response.json()["id"] == encoded_id
def _mistral_plus_anthropic_router() -> Router:
return Router(
model_list=[
{
"model_name": "mistral-ocr",
"litellm_params": {"model": "mistral/mistral-ocr-latest", "api_key": "mistral-key"},
"model_info": {"id": "mistral-ocr-id"},
},
{
"model_name": "claude-opus-4-6",
"litellm_params": {"model": "anthropic/claude-opus-4-6", "api_key": "anthropic-key"},
"model_info": {"id": "claude-id"},
},
]
)
def _restricted_key(key_models: list[str]) -> UserAPIKeyAuth:
from litellm.proxy._types import LitellmUserRoles
return UserAPIKeyAuth(
api_key="test-key",
user_role=LitellmUserRoles.INTERNAL_USER,
user_id="test-user",
team_id="team-a",
team_models=["claude-opus-4-6", "mistral-ocr"],
models=key_models,
)
@pytest.mark.parametrize(
"http_method, path_suffix, litellm_fn",
[
("get", "", "afile_retrieve"),
("get", "/content", "afile_content"),
("delete", "", "afile_delete"),
],
)
def test_model_routed_file_ops_reject_key_without_model_grant(
mocker: MockerFixture, monkeypatch, http_method: str, path_suffix: str, litellm_fn: str
):
"""
Regression: a key whose allowlist does not include the deployment named in a
model-encoded file id must be refused before that deployment's server-side
credentials are resolved. Previously any key could name any deployment via the
id (or the x-litellm-model header) and act on that provider account's files.
"""
import litellm.proxy.proxy_server as ps
from litellm.proxy.openai_files_endpoints.common_utils import encode_file_id_with_model
router = _mistral_plus_anthropic_router()
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router)
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router)
proxy_logging_obj.update_request_status = mocker.AsyncMock()
proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock()
upstream = mocker.AsyncMock(side_effect=AssertionError("provider must not be called"))
monkeypatch.setattr(litellm, litellm_fn, upstream)
app.dependency_overrides[ps.user_api_key_auth] = lambda: _restricted_key(["claude-opus-4-6"])
encoded_id = encode_file_id_with_model("7a13fa8e-fcf8-42c5-aa61-c93c10e2c7df", "mistral-ocr")
try:
response = getattr(client, http_method)(
f"/v1/files/{encoded_id}{path_suffix}", headers={"Authorization": "Bearer test-key"}
)
finally:
app.dependency_overrides.pop(ps.user_api_key_auth, None)
assert response.status_code == 403, response.text
assert response.json()["error"]["type"] == "key_model_access_denied"
upstream.assert_not_called()
def test_list_files_header_model_rejects_key_without_model_grant(mocker: MockerFixture, monkeypatch):
import litellm.proxy.proxy_server as ps
router = _mistral_plus_anthropic_router()
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router)
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router)
proxy_logging_obj.update_request_status = mocker.AsyncMock()
proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock()
upstream = mocker.AsyncMock(side_effect=AssertionError("provider must not be called"))
monkeypatch.setattr(litellm, "afile_list", upstream)
app.dependency_overrides[ps.user_api_key_auth] = lambda: _restricted_key(["claude-opus-4-6"])
try:
response = client.get(
"/v1/files", headers={"Authorization": "Bearer test-key", "x-litellm-model": "mistral-ocr"}
)
finally:
app.dependency_overrides.pop(ps.user_api_key_auth, None)
assert response.status_code == 403, response.text
upstream.assert_not_called()
def test_model_routed_file_retrieve_allows_key_with_model_grant(mocker: MockerFixture, monkeypatch):
"""The grant check must not break the happy path: a key allowed the deployment still resolves its credentials."""
import litellm.proxy.proxy_server as ps
from litellm.proxy.openai_files_endpoints.common_utils import encode_file_id_with_model
router = _mistral_plus_anthropic_router()
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router)
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router)
proxy_logging_obj.update_request_status = mocker.AsyncMock()
proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock()
captured_kwargs: dict = {}
async def _mock_afile_retrieve(**kwargs):
captured_kwargs.update(kwargs)
return OpenAIFileObject(
id="7a13fa8e-fcf8-42c5-aa61-c93c10e2c7df",
object="file",
bytes=2,
created_at=1234567890,
filename="batch.jsonl",
purpose="batch",
status="uploaded",
)
monkeypatch.setattr(litellm, "afile_retrieve", _mock_afile_retrieve)
app.dependency_overrides[ps.user_api_key_auth] = lambda: _restricted_key(["mistral-ocr"])
encoded_id = encode_file_id_with_model("7a13fa8e-fcf8-42c5-aa61-c93c10e2c7df", "mistral-ocr")
try:
response = client.get(f"/v1/files/{encoded_id}", headers={"Authorization": "Bearer test-key"})
finally:
app.dependency_overrides.pop(ps.user_api_key_auth, None)
assert response.status_code == 200, response.text
assert captured_kwargs["api_key"] == "mistral-key"
assert captured_kwargs["custom_llm_provider"] == "mistral"

1
tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py
View file

@ -384,6 +384,7 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset(
"tencent",
"tensormesh",
"text-completion-inception",
"transcribe",
"valkey",
"xiaomi_mimo",
"zai",

60
tests/test_litellm/proxy/test_batch_x_litellm_model_encoding.py
View file

@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.openai_files_endpoints.common_utils import (
decode_model_from_file_id,
get_batch_id_from_unified_batch_id,
@ -58,10 +59,7 @@ def _make_batch_response(
def test_get_batch_id_from_unified_batch_id_handles_appended_fields():
decoded_id = (
"litellm_proxy;model_id:deployment-123;"
"llm_batch_id:batch_openai_123;llm_output_file_id:file-output"
)
decoded_id = "litellm_proxy;model_id:deployment-123;llm_batch_id:batch_openai_123;llm_output_file_id:file-output"
assert get_batch_id_from_unified_batch_id(decoded_id) == "batch_openai_123"
@ -107,12 +105,10 @@ async def test_create_batch_with_x_litellm_model_encodes_batch_id():
}
),
),
patch("litellm.proxy.batches_endpoints.endpoints.ProxyBaseLLMRequestProcessing") as mock_processor_cls,
patch(
"litellm.proxy.batches_endpoints.endpoints.ProxyBaseLLMRequestProcessing"
) as mock_processor_cls,
patch(
"litellm.proxy.batches_endpoints.endpoints.get_credentials_for_model",
return_value=mock_credentials,
"litellm.proxy.batches_endpoints.endpoints.get_authorized_credentials_for_model",
new=AsyncMock(return_value=mock_credentials),
),
patch(
"litellm.proxy.batches_endpoints.endpoints.prepare_data_with_credentials",
@ -165,23 +161,15 @@ async def test_create_batch_with_x_litellm_model_encodes_batch_id():
)
# The batch_id should be encoded with model info
assert (
response.id != raw_batch_id
), f"Expected batch_id to be encoded, but got raw ID: {response.id}"
assert response.id.startswith(
"batch_"
), f"Encoded batch_id should keep batch_ prefix, got: {response.id}"
assert response.id != raw_batch_id, f"Expected batch_id to be encoded, but got raw ID: {response.id}"
assert response.id.startswith("batch_"), f"Encoded batch_id should keep batch_ prefix, got: {response.id}"
# Should be decodable back to the original
decoded_model = decode_model_from_file_id(response.id)
assert (
decoded_model == model_name
), f"Expected model '{model_name}' from decoded batch_id, got: {decoded_model}"
assert decoded_model == model_name, f"Expected model '{model_name}' from decoded batch_id, got: {decoded_model}"
original_id = get_original_file_id(response.id)
assert (
original_id == raw_batch_id
), f"Expected original ID '{raw_batch_id}', got: {original_id}"
assert original_id == raw_batch_id, f"Expected original ID '{raw_batch_id}', got: {original_id}"
assert mock_create_batch.call_args.kwargs["metadata"] == {"customer_id": "cust-123"}
@ -227,12 +215,10 @@ async def test_create_batch_with_x_litellm_model_encodes_output_and_error_file_i
}
),
),
patch("litellm.proxy.batches_endpoints.endpoints.ProxyBaseLLMRequestProcessing") as mock_processor_cls,
patch(
"litellm.proxy.batches_endpoints.endpoints.ProxyBaseLLMRequestProcessing"
) as mock_processor_cls,
patch(
"litellm.proxy.batches_endpoints.endpoints.get_credentials_for_model",
return_value=mock_credentials,
"litellm.proxy.batches_endpoints.endpoints.get_authorized_credentials_for_model",
new=AsyncMock(return_value=mock_credentials),
),
patch(
"litellm.proxy.batches_endpoints.endpoints.prepare_data_with_credentials",
@ -316,9 +302,7 @@ async def test_create_batch_without_x_litellm_model_returns_raw_ids(monkeypatch)
}
),
),
patch(
"litellm.proxy.batches_endpoints.endpoints.ProxyBaseLLMRequestProcessing"
) as mock_processor_cls,
patch("litellm.proxy.batches_endpoints.endpoints.ProxyBaseLLMRequestProcessing") as mock_processor_cls,
patch(
"litellm.acreate_batch",
new=AsyncMock(return_value=mock_response),
@ -383,9 +367,7 @@ class TestBatchIdRoundTripWithRetrieve:
raw_batch_id = "batch_vllm_12345"
# What create_batch does:
encoded_id = encode_file_id_with_model(
file_id=raw_batch_id, model=model_name, id_type="batch"
)
encoded_id = encode_file_id_with_model(file_id=raw_batch_id, model=model_name, id_type="batch")
# What retrieve_batch does:
decoded_model = decode_model_from_file_id(encoded_id)
@ -410,9 +392,7 @@ class TestBatchIdRoundTripWithRetrieve:
]
for raw_id, model in test_cases:
encoded = encode_file_id_with_model(
file_id=raw_id, model=model, id_type="batch"
)
encoded = encode_file_id_with_model(file_id=raw_id, model=model, id_type="batch")
assert encoded.startswith("batch_")
assert decode_model_from_file_id(encoded) == model
assert get_original_file_id(encoded) == raw_id
@ -433,16 +413,10 @@ async def test_cancel_batch_with_unified_id_routes_with_decoded_model_and_batch_
mock_request.url.path = f"/v1/batches/{unified_batch_id}/cancel"
mock_fastapi_response = MagicMock()
mock_fastapi_response.headers = {}
mock_user_api_key_dict = MagicMock()
mock_user_api_key_dict.parent_otel_span = None
mock_user_api_key_dict.user_id = "test_user"
mock_user_api_key_dict.allowed_model_region = None
mock_user_api_key_dict.team_metadata = {}
mock_user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="test_user", team_metadata={})
with (
patch(
"litellm.proxy.batches_endpoints.endpoints.ProxyBaseLLMRequestProcessing"
) as mock_processor_cls,
patch("litellm.proxy.batches_endpoints.endpoints.ProxyBaseLLMRequestProcessing") as mock_processor_cls,
patch(
"litellm.proxy.batches_endpoints.endpoints.update_batch_in_database",
new=AsyncMock(),

72
tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
View file

@ -306,6 +306,40 @@ async def test_vector_store_file_list_resolves_credentials_from_model_query_para
)
@pytest.mark.asyncio
async def test_vector_store_file_list_registry_routed_model_skips_key_model_grant():
request = MagicMock(spec=Request)
request.query_params = {}
request.headers = {}
llm_router = MagicMock()
llm_router.get_deployment_credentials_with_provider.return_value = {
"api_key": "sk-team-openai",
"api_base": "https://api.openai.com/v1",
"custom_llm_provider": "openai",
"model": "openai/gpt-4o-mini",
}
data = {"vector_store_id": "vs_123", "model": "team-openai"}
user_api_key_dict = UserAPIKeyAuth(
models=["restricted-deployment"],
team_models=["restricted-deployment"],
)
result = await _update_request_data_with_model_routing_hint(
data=data,
request=request,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
)
assert result["api_key"] == "sk-team-openai"
assert result["model"] == "openai/gpt-4o-mini"
llm_router.get_deployment_credentials_with_provider.assert_called_once_with(
model_id="team-openai"
)
@pytest.mark.asyncio
async def test_vector_store_file_list_resolves_single_openai_team_deployment():
request = MagicMock(spec=Request)
@ -575,6 +609,44 @@ async def test_vector_store_file_list_authorizes_model_query_param_before_creden
llm_router.get_deployment_credentials_with_provider.assert_not_called()
@pytest.mark.asyncio
async def test_vector_store_file_list_model_query_param_enforces_project_model_grant():
from litellm.proxy._types import LiteLLM_ProjectTableCachedObj, LiteLLM_TeamTableCachedObj
from litellm.proxy.auth.auth_checks import ProxyException
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key
request = MagicMock(spec=Request)
request.query_params = {"model": "team-openai"}
request.headers = {}
llm_router = MagicMock()
llm_router.model_group_alias = {}
cache = UserApiKeyCache()
await cache.async_set_cache(
key="team_id:team-123",
value=LiteLLM_TeamTableCachedObj(team_id="team-123", models=["team-openai"]),
)
await cache.async_set_cache(
key=project_cache_key("proj-1"),
value=LiteLLM_ProjectTableCachedObj(project_id="proj-1", models=["other-deployment"]),
)
user_api_key_dict = UserAPIKeyAuth(team_id="team-123", team_models=["team-openai"], project_id="proj-1")
with (
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: proxy_server global, no seam
patch("litellm.proxy.proxy_server.user_api_key_cache", cache), # test-quality-ok: proxy_server global, no seam
):
with pytest.raises(ProxyException):
await _update_request_data_with_model_routing_hint(
data={"vector_store_id": "vs_123"},
request=request,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
)
llm_router.get_deployment_credentials_with_provider.assert_not_called()
@pytest.mark.asyncio
async def test_update_request_data_with_litellm_managed_vector_store_registry():
"""

1
tests/test_litellm/rerank_api/test_main.py
View file

@ -239,6 +239,7 @@ async def test_arerank_error_is_mapped_to_litellm_exception(respx_mock: respx.Mo
@pytest.mark.asyncio
@pytest.mark.timeout(300)
async def test_arerank_declared_authenticating_provider_skips_resolution(monkeypatch):
"""Regression for the event-loop hazard in arerank's provider pre-resolution:
get_llm_provider runs the blocking OAuth device flow for github_copilot/chatgpt,

47
tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -2917,6 +2917,39 @@ class TestUsageTransformation:
assert getattr(response_usage.input_tokens_details, "cache_write_tokens", None) == 800
assert response_usage.input_tokens_details.model_dump()["cache_write_tokens"] == 800
def test_transform_usage_preserves_input_modality_tokens(self):
"""Regression: the bridge dropped image and video input tokens.
Vertex reports prompt tokens split by modality, so a Live session that sends
camera frames arrives with image_tokens set. InputTokensDetails declared only
audio/cached/text, so those tokens were folded into text and lost their
attribution, and any per-modality rate could never apply to them.
"""
usage = Usage(
prompt_tokens=300,
completion_tokens=10,
total_tokens=310,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=20, audio_tokens=80, image_tokens=150, video_tokens=50, cached_tokens=0
),
completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=10),
)
response_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(
chat_completion_response=usage
)
details = response_usage.input_tokens_details
assert details is not None
assert getattr(details, "image_tokens", None) == 150
assert getattr(details, "video_tokens", None) == 50
assert getattr(details, "audio_tokens", None) == 80
from litellm.responses.utils import ResponseAPILoggingUtils
back = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(response_usage.model_dump())
assert back.prompt_tokens_details.image_tokens == 150
assert back.prompt_tokens_details.video_tokens == 50
def test_transform_usage_with_reasoning_tokens_gemini(self):
"""Test that reasoning_tokens from Gemini are properly transformed to output_tokens_details"""
# Setup: Simulate Gemini usage with thoughtsTokenCount
@ -5000,3 +5033,17 @@ def test_transform_chat_completion_response_incomplete_details():
assert result_existing.status == "incomplete"
assert result_existing.incomplete_details == existing_details
@pytest.mark.parametrize("stream", [True, False])
async def test_bridge_rejects_untranslatable_tool_choice_with_a_400(stream: bool):
with pytest.raises(litellm.BadRequestError) as exc_info:
await litellm.aresponses(
model="anthropic/claude-haiku-4-5",
input="Which fruit is red?",
tools=[{"type": "function", "name": "lookup_fruit", "parameters": {"type": "object"}}],
tool_choice={"type": "file_search"},
stream=stream,
api_key="sk-unused",
)
assert exc_info.value.status_code == 400
assert "tool_choice={'type': 'file_search'}" in str(exc_info.value)

14
tests/test_litellm/test_main.py
View file

@ -3967,3 +3967,17 @@ def test_aiohttp_openai_warns_only_when_http2_enabled(
assert handler_completion.called
warned: Final = "aiohttp_openai/ always uses aiohttp" in caplog.text
assert warned is http2_on
@pytest.mark.parametrize("tool_choice", [{"type": "bogus"}, {"name": "lookup_fruit"}, {"type": "file_search"}])
def test_completion_rejects_untranslatable_tool_choice_with_a_400(tool_choice):
with pytest.raises(litellm.BadRequestError) as exc_info:
litellm.completion(
model="anthropic/claude-haiku-4-5",
messages=[{"role": "user", "content": "Which fruit is red?"}],
tools=[{"type": "function", "function": {"name": "lookup_fruit", "parameters": {"type": "object"}}}],
tool_choice=tool_choice,
api_key="sk-unused",
)
assert exc_info.value.status_code == 400
assert f"tool_choice={tool_choice}" in str(exc_info.value)

74
tests/test_litellm/test_model_prices_schema.py
View file

@ -11,7 +11,9 @@ from typing import Final
import jsonschema
import pytest
import litellm
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
REPO_ROOT = Path(__file__).parents[2]
@ -363,3 +365,75 @@ def test_active_mistral_chat_rows_price_cache_reads_below_input(path: Path):
and not cache_read_is_tenth_of_input(entry)
]
assert drifted == []
PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset({"sagemaker", "bedrock_converse"})
MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "evaluation"})
VERTEX_FAMILIES_A_VERTEX_WILDCARD_GRANT_DOES_NOT_LIST: Final = frozenset(
{
"vertex_ai-ai21_models",
"vertex_ai-embedding-models",
"vertex_ai-image-models",
"vertex_ai-llama_models",
"vertex_ai-mistral_models",
"vertex_ai-openai_models",
"vertex_ai-qwen_models",
"vertex_ai-video-models",
}
)
def is_registered_provider(label: str, model_names: tuple[str, ...]) -> bool:
if label in litellm.models_by_provider or JSONProviderRegistry.exists(label):
return True
family_root: Final = label.split("-", 1)[0]
wildcard_models: Final = litellm.models_by_provider.get(family_root, ())
return any(
name in wildcard_models or name.removeprefix(f"{family_root}/") in wildcard_models for name in model_names
)
def unregistered_providers(rows: Mapping[str, object]) -> list[str]:
labelled_rows: Final = tuple(
(name, entry["litellm_provider"])
for name, entry in rows.items()
if name != "sample_spec"
and isinstance(entry, dict)
and "litellm_provider" in entry
and entry.get("mode") not in MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY
and entry["litellm_provider"] not in PROVIDER_LABELS_WITHOUT_A_MODEL_SET
and entry["litellm_provider"] not in VERTEX_FAMILIES_A_VERTEX_WILDCARD_GRANT_DOES_NOT_LIST
)
return sorted(
label
for label in {label for _, label in labelled_rows}
if not is_registered_provider(label, tuple(name for name, row_label in labelled_rows if row_label == label))
)
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
def test_every_cost_map_provider_is_registered(path: Path):
assert unregistered_providers(json.loads(path.read_text())) == [], (
f"{path.name} carries a litellm_provider whose models a `<provider>/*` grant does not list. A new provider "
"needs a `<provider>_models` set in litellm/__init__.py, filled in _populate_provider_model_sets and listed "
"in _build_models_by_provider. A new `<provider>-<family>` label needs its rows added to a set that "
"`models_by_provider[<provider>]` includes"
)
def test_unregistered_provider_guard_flags_only_labels_nobody_registered():
wired_vertex_model: Final = sorted(litellm.vertex_language_models)[0]
rows: Final = {
"sample_spec": {"litellm_provider": "one of the supported providers", "mode": "chat"},
"nobody_registered/StartJob": {"litellm_provider": "nobody_registered", "mode": "audio_transcription"},
"gpt-4o": {"litellm_provider": "openai", "mode": "chat"},
wired_vertex_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"},
"vertex_ai/new-family-model": {"litellm_provider": "vertex_ai-new_family_models", "mode": "chat"},
"unknown_root/model": {"litellm_provider": "unknown_root-new_family_models", "mode": "chat"},
"some_search/search": {"litellm_provider": "some_search", "mode": "search"},
}
assert unregistered_providers(rows) == [
"nobody_registered",
"unknown_root-new_family_models",
"vertex_ai-new_family_models",
]

87
tests/test_litellm/test_router.py
View file

@ -5745,6 +5745,93 @@ async def test_router_unknown_model_error_message_renders_model_name_literally()
assert " " not in message # no padding run from an expanded format field
def test_get_credential_deployment_is_the_deployment_credentials_resolve_to():
"""Regression: a batch retrieved with credentials resolved by model name was priced
without its deployment id, so per-deployment pricing never applied. The deployment
behind the credentials must be reachable by name and by id, carrying its model_info."""
router = litellm.Router(
model_list=[
{
"model_name": "mistral-ocr",
"litellm_params": {"model": "mistral/mistral-ocr-latest", "api_key": "sk-ocr"},
"model_info": {"id": "ocr-dep", "ocr_cost_per_page_batches": 0.0123},
}
]
)
by_name = router.get_credential_deployment(model_id="mistral-ocr")
by_id = router.get_credential_deployment(model_id="ocr-dep")
assert by_name is not None and by_id is not None
assert by_name.model_info.id == by_id.model_info.id == "ocr-dep"
assert by_name.model_info.model_dump()["ocr_cost_per_page_batches"] == 0.0123
assert router.get_deployment_credentials_with_provider(model_id="mistral-ocr")["api_key"] == "sk-ocr"
assert router.get_credential_deployment(model_id="no-such-model") is None
def test_get_credential_deployment_skips_a_paused_deployment():
router = litellm.Router(
model_list=[
{
"model_name": "paused-ocr",
"litellm_params": {"model": "mistral/mistral-ocr-latest", "api_key": "sk-ocr"},
"model_info": {"id": "paused-dep", "blocked": True},
}
]
)
assert router.get_credential_deployment(model_id="paused-ocr") is None
assert router.get_credential_deployment(model_id="paused-dep") is None
def test_get_team_public_name_deployment_only_resolves_the_owning_team():
router = litellm.Router(
model_list=[
{
"model_name": "mistral/mistral-ocr-latest",
"litellm_params": {"model": "mistral/mistral-ocr-latest", "api_key": "sk-team-a"},
"model_info": {"id": "team-a-ocr", "team_id": "team-a", "team_public_model_name": "ocr"},
}
]
)
owning_team = router._get_team_public_name_deployment(model_id="ocr", team_id="team-a")
assert owning_team is not None and owning_team.model_info.id == "team-a-ocr"
assert router._get_team_public_name_deployment(model_id="ocr", team_id="team-b") is None
assert router._get_team_public_name_deployment(model_id="ocr", team_id=None) is None
assert router.get_credential_deployment(model_id="ocr", team_id="team-a").model_info.id == "team-a-ocr"
assert router.get_credential_deployment(model_id="ocr", team_id="team-b") is None
def test_get_wildcard_deployment_usable_by_team_prefers_the_team_pattern():
router = litellm.Router(
model_list=[
{
"model_name": "mistral/*",
"litellm_params": {"model": "mistral/*", "api_key": "sk-shared"},
"model_info": {"id": "shared-wildcard"},
},
{
"model_name": "mistral/*",
"litellm_params": {"model": "mistral/*", "api_key": "sk-team-a"},
"model_info": {"id": "team-a-wildcard", "team_id": "team-a", "team_public_model_name": "mistral/*"},
},
]
)
ocr = "mistral/mistral-ocr-latest"
team_match = router._get_wildcard_deployment_usable_by_team(model_id=ocr, team_id="team-a")
other_team_match = router._get_wildcard_deployment_usable_by_team(model_id=ocr, team_id="team-b")
anonymous_match = router._get_wildcard_deployment_usable_by_team(model_id=ocr, team_id=None)
assert team_match is not None and team_match.model_info.id == "team-a-wildcard"
assert other_team_match is not None and other_team_match.model_info.id == "shared-wildcard"
assert anonymous_match is not None and anonymous_match.model_info.id == "shared-wildcard"
assert router._get_wildcard_deployment_usable_by_team(model_id="openai/gpt-5.6", team_id="team-a") is None
assert router.get_credential_deployment(model_id=ocr, team_id="team-b").model_info.id == "shared-wildcard"
def test_get_deployment_credentials_with_provider_aws_bedrock_runtime_endpoint():
"""
Test that get_deployment_credentials_with_provider correctly copies

83
tests/test_litellm/test_unit_shard_per_test_timeout.py Normal file
View file

@ -0,0 +1,83 @@
import shlex
import subprocess
import sys
from pathlib import Path
from string import Template
from types import MappingProxyType
from typing import Final
import pytest
import yaml
_REPO_ROOT: Final = Path(__file__).resolve().parents[2]
_BASE_WORKFLOW: Final = _REPO_ROOT / ".github" / "workflows" / "_test-unit-base.yml"
_SHARD_ENV: Final = MappingProxyType({"WORKERS": "2", "RERUNS": "2", "DIST": "loadscope", "TEST_TIMEOUT_SECONDS": "1"})
_HANG_GUARD_FLAGS: Final = frozenset(("-n", "--dist", "--reruns", "--reruns-delay", "--timeout", "--rerun-except"))
_HUNG_TEST_MODULE: Final = """
import threading
import pytest
@pytest.fixture
def hangs_on_teardown():
yield
threading.Event().wait()
def test_body_waits_forever():
threading.Event().wait()
def test_fixture_teardown_waits_forever(hangs_on_teardown):
assert True
def test_passes():
assert True
"""
def _run_tests_script() -> str:
workflow: Final = yaml.safe_load(_BASE_WORKFLOW.read_text())
return next(step["run"] for step in workflow["jobs"]["run"]["steps"] if step.get("name") == "Run tests")
def _pytest_invocations(script: str) -> tuple[tuple[str, ...], ...]:
return tuple(tuple(shlex.split(line)) for line in script.replace("\\\n", " ").splitlines() if " pytest " in line)
def _hang_guard_args(invocation: tuple[str, ...]) -> tuple[str, ...]:
return tuple(
Template(token).safe_substitute(_SHARD_ENV)
for previous, token in zip(("", *invocation), invocation)
if token.split("=", 1)[0] in _HANG_GUARD_FLAGS or previous in _HANG_GUARD_FLAGS
)
_INVOCATIONS: Final = _pytest_invocations(_run_tests_script())
@pytest.mark.parametrize(
"invocation", _INVOCATIONS, ids=tuple("xdist" if "-n" in invocation else "serial" for invocation in _INVOCATIONS)
)
def test_a_hung_test_fails_fast_and_names_itself_under_the_shard_flags(
invocation: tuple[str, ...], tmp_path: Path
) -> None:
hung_module: Final = tmp_path / "test_hung.py"
hung_module.write_text(_HUNG_TEST_MODULE)
result: Final = subprocess.run(
(sys.executable, "-m", "pytest", str(hung_module), "-p", "no:cacheprovider", *_hang_guard_args(invocation)),
cwd=tmp_path,
capture_output=True,
text=True,
timeout=90,
check=False,
)
assert result.returncode == 1, result.stdout
assert "FAILED test_hung.py::test_body_waits_forever" in result.stdout
assert "ERROR test_hung.py::test_fixture_teardown_waits_forever" in result.stdout
assert "Timeout (>1.0s) from pytest-timeout" in result.stdout
assert "1 failed, 2 passed, 1 error" in result.stdout

2
tests/test_litellm/test_utils.py
View file

@ -755,7 +755,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"input_cost_per_video_per_second_above_128k_tokens": {"type": "number"},
"input_dbu_cost_per_token": {"type": "number"},
"annotation_cost_per_page": {"type": "number"},
"annotation_cost_per_page_batches": {"type": "number"},
"ocr_cost_per_page": {"type": "number"},
"ocr_cost_per_page_batches": {"type": "number"},
"ocr_cost_per_credit": {"type": "number"},
"code_interpreter_cost_per_session": {"type": "number"},
"inference_geo": {"type": "string"},

11
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -30665,6 +30665,8 @@ export interface components {
allow_client_keepalive_override: boolean | null;
/** Annotation Cost Per Page */
annotation_cost_per_page?: number | null;
/** Annotation Cost Per Page Batches */
annotation_cost_per_page_batches?: number | null;
/** Api Base */
api_base?: string | null;
/** Api Key */
@ -30894,6 +30896,8 @@ export interface components {
ocr_cost_per_credit?: number | null;
/** Ocr Cost Per Page */
ocr_cost_per_page?: number | null;
/** Ocr Cost Per Page Batches */
ocr_cost_per_page_batches?: number | null;
/** Organization */
organization?: string | null;
/** Otpm */
@ -31795,9 +31799,8 @@ export interface components {
/**
* Api Version
* @description API version for Javelin service
* @default v1
*/
api_version: string | null;
api_version?: string | null;
/**
* Application
* @description Application name for Javelin service
@ -41316,6 +41319,8 @@ export interface components {
allow_client_keepalive_override: boolean | null;
/** Annotation Cost Per Page */
annotation_cost_per_page?: number | null;
/** Annotation Cost Per Page Batches */
annotation_cost_per_page_batches?: number | null;
/** Api Base */
api_base?: string | null;
/** Api Key */
@ -41545,6 +41550,8 @@ export interface components {
ocr_cost_per_credit?: number | null;
/** Ocr Cost Per Page */
ocr_cost_per_page?: number | null;
/** Ocr Cost Per Page Batches */
ocr_cost_per_page_batches?: number | null;
/** Organization */
organization?: string | null;
/** Otpm */