scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_techdebt_20260901

Browse source
This commit is contained in:
mateo-berri 2026-09-02 10:59:17 -07:00
commit dcc2c2ac3a
32 changed files with 1355 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/Dockerfile
View file

@ -46,6 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3.13
# Stage 2 — copy source and install the project + workspace members.
@ -57,6 +58,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3.13
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

2
helm/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 1.1.2
version: 1.1.3
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

42
helm/litellm-helm/tests/hpa_tests.yaml
View file

@ -1,4 +1,4 @@
suite: "hpa with behavior"
suite: "hpa"
templates:
- hpa.yaml
tests:
@ -23,14 +23,44 @@ tests:
- equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 }
- equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 }
---
suite: "hpa without behavior"
templates:
- hpa.yaml
tests:
- it: "does not render behavior when not set"
set:
autoscaling.enabled: true
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- isNull: { path: spec.behavior }
- it: "scales on cpu at the documented 60 percent by default"
set:
autoscaling.enabled: true
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
- equal: { path: "spec.metrics[0].resource.target.type", value: Utilization }
- equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 }
- it: "does not scale on memory by default"
set:
autoscaling.enabled: true
asserts:
- lengthEqual: { path: spec.metrics, count: 1 }
- it: "honours an explicit cpu target override"
set:
autoscaling.enabled: true
autoscaling.targetCPUUtilizationPercentage: 75
asserts:
- equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 }
- it: "renders a memory metric only when a memory target is set"
set:
autoscaling.enabled: true
autoscaling.targetMemoryUtilizationPercentage: 80
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- equal: { path: "spec.metrics[1].resource.name", value: memory }
- equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 }
- it: "renders no hpa when autoscaling is disabled"
asserts:
- hasDocuments: { count: 0 }

11
helm/litellm-helm/values.yaml
View file

@ -200,7 +200,16 @@ autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
# 60 is the documented recommendation. See "Recommended Machine Specifications"
# in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe
# above only after up to failureThreshold x periodSeconds = 300 seconds, so a target
# high enough to trip near saturation adds capacity minutes after it was needed.
targetCPUUtilizationPercentage: 60
# Deliberately left unset rather than given a value. The prisma query engine's
# resident memory is a high-water mark that ratchets to the pod's worst-ever write
# and is never returned, so a memory target reads the largest write a pod ever did
# rather than what it is doing now, and replicas ratchet up without scaling back in.
# Memory is a floor to provision under 'resources', not a signal to scale on.
# targetMemoryUtilizationPercentage: 80
# behavior: {}

65
litellm/integrations/custom_guardrail.py
View file

@ -1,4 +1,5 @@
import contextvars
import copy
import hashlib
import os
import secrets
@ -39,6 +40,7 @@ except ImportError:
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
dc: Final = DualCache()
@ -852,6 +854,69 @@ class CustomGuardrail(CustomLogger):
return result
async def async_logging_hook(
self,
kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract
result: object,
call_type: str,
) -> tuple[dict, object]: # mutable-ok: CustomLogger.async_logging_hook contract
"""logging_only: run apply_guardrail on copies of the logged request/response and record the verdict."""
from litellm.llms import get_guardrail_translation_mapping
if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks:
return kwargs, result
try:
translation: Final = get_guardrail_translation_mapping(CallTypes(call_type))()
except ValueError:
verbose_logger.debug(
"Guardrail %s: no guardrail translation for call_type=%s, skipping logging_only scan",
self.guardrail_name,
call_type,
)
return kwargs, result
litellm_params: Final = kwargs.get("litellm_params") or {}
scratch_metadata: Final = {
key: value
for key, value in (litellm_params.get("metadata") or {}).items()
if key != "standard_logging_guardrail_information"
}
try:
await self._scan_logged_call(kwargs, result, translation, scratch_metadata)
except Exception as e:
verbose_logger.warning("Guardrail %s: logging_only scan raised: %s", self.guardrail_name, e)
recorded: Final = scratch_metadata.get("standard_logging_guardrail_information")
standard_logging_object: Final = kwargs.get("standard_logging_object")
if not recorded or not isinstance(standard_logging_object, dict):
return kwargs, result
entries: Final = recorded if isinstance(recorded, list) else [recorded]
existing: Final = standard_logging_object.get("guardrail_information") or []
return {
**kwargs,
"standard_logging_object": {**standard_logging_object, "guardrail_information": [*existing, *entries]},
}, result
async def _scan_logged_call(
self,
kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract
result: object,
translation: "BaseTranslation",
scratch_metadata: dict, # mutable-ok: apply_guardrail records its verdict into request metadata
) -> None:
optional_params: Final = kwargs.get("optional_params") or {}
scratch_input: Final = copy.deepcopy(kwargs.get("messages") or kwargs.get("input"))
scratch_request: Final = {
"model": kwargs.get("model"),
"messages": scratch_input,
"input": scratch_input,
"tools": copy.deepcopy(optional_params.get("tools")),
"litellm_call_id": kwargs.get("litellm_call_id"),
"metadata": scratch_metadata,
}
await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self)
await translation.process_output_response(
response=copy.deepcopy(result), guardrail_to_apply=self, request_data=scratch_request
)
def supports_scan_only_tool_results(self) -> bool:
"""Whether this guardrail can scan tool-result content.

8
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -36,6 +36,8 @@ from litellm.types.utils import (
from litellm.utils import print_verbose, token_counter
if TYPE_CHECKING:
from openai.types.completion_usage import CompletionUsage
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.litellm_core_utils.streaming_chunk_builder_utils import (
UsagePerChunk,
@ -794,7 +796,7 @@ class ChunkProcessor:
@staticmethod
def _extract_usage_chunk(chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream") -> Usage | None:
usage_chunk: Usage | None = None
usage_chunk: Usage | CompletionUsage | None = None
if hasattr(chunk, "usage") and chunk.usage is not None:
usage_chunk = chunk.usage
elif "usage" in chunk:
@ -806,7 +808,9 @@ class ChunkProcessor:
if isinstance(usage_chunk, dict):
return Usage(**usage_chunk)
return usage_chunk
if usage_chunk is None or isinstance(usage_chunk, Usage):
return usage_chunk
return Usage(**usage_chunk.model_dump())
def _calculate_usage_per_chunk(
self,

17
litellm/llms/anthropic/common_utils.py
View file

@ -1378,31 +1378,38 @@ def process_anthropic_headers(headers: httpx.Headers | dict) -> dict:
return additional_headers
def _anthropic_model_entry(model: ModelInfoResponse, created_at: str) -> Mapping[str, object]:
def _anthropic_model_entry(
model: ModelInfoResponse, created_at: str, display_names: Mapping[str, str]
) -> Mapping[str, object]:
return { # mutable-ok: JSON response body, serialized by the route and never mutated
"type": "model",
"id": model["id"],
"display_name": model["id"],
"display_name": display_names.get(model["id"], model["id"]),
"created_at": created_at,
"max_input_tokens": model.get("max_input_tokens"),
"max_tokens": model.get("max_output_tokens"),
}
def create_anthropic_model_list_response(models: Sequence[ModelInfoResponse]) -> Mapping[str, object]:
def create_anthropic_model_list_response(
models: Sequence[ModelInfoResponse],
display_names: Mapping[str, str] = MappingProxyType({}),
) -> Mapping[str, object]:
"""Build the Anthropic-native /v1/models envelope.
Clients that send an anthropic-version header parse the Anthropic Models API
shape (type/display_name/created_at plus has_more/first_id/last_id) and filter
the list themselves, so every model is returned here. The token limits carry
over from the OpenAI-shaped listing, named as the Messages API names them, and
are always present because the vendor shape declares them nullable, not optional
are always present because the vendor shape declares them nullable, not optional.
display_names maps a listed model id to a configured human-readable name; ids
without an entry fall back to the id itself, matching the vendor behavior
"""
created_at: Final = (
datetime.fromtimestamp(DEFAULT_MODEL_CREATED_AT_TIME, tz=timezone.utc).isoformat().replace("+00:00", "Z")
)
data: Final = [ # mutable-ok: JSON response body, serialized by the route and never mutated
_anthropic_model_entry(model, created_at) for model in models
_anthropic_model_entry(model, created_at, display_names) for model in models
]
return { # mutable-ok: JSON response body, serialized by the route and never mutated
"data": data,

2
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -949,7 +949,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# For Gemini 3+ models, use thinkingLevel instead of thinkingBudget
if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
if thinking_enabled:
if thinking_budget is None or thinking_budget == 0:
if thinking_budget == 0:
params["includeThoughts"] = False
else:
params["includeThoughts"] = True

3
litellm/llms/vertex_ai/ocr/deepseek_transformation.py
View file

@ -177,8 +177,9 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig):
content_item = {"type": "image_url", "image_url": document_url}
# Build DeepSeek OCR request
provider_model: Final = model if model.startswith("deepseek-ai/") else f"deepseek-ai/{model}"
data: Final = {
"model": "deepseek-ai/" + model,
"model": provider_model,
"messages": [{"role": "user", "content": [content_item]}],
}

22
litellm/main.py
View file

@ -8637,6 +8637,16 @@ def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Opti
hidden_params["response_cost"] = response_cost
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if isinstance(getattr(usage, "cost", None), (int, float)):
return
computed_cost: Final = logging_obj._response_cost_calculator(result=response)
if isinstance(computed_cost, (int, float)) and computed_cost > 0:
setattr(usage, "cost", computed_cost)
def stream_chunk_builder(
chunks: list,
messages: list | None = None,
@ -8731,12 +8741,7 @@ def stream_chunk_builder(
)
break
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
setattr(
usage,
"cost",
logging_obj._response_cost_calculator(result=response),
)
_stamp_streaming_usage_cost(usage, response, logging_obj)
_set_stream_builder_response_cost(response, logging_obj)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
@ -8915,10 +8920,7 @@ def stream_chunk_builder(
)
break
# Add cost to usage object if include_cost_in_streaming_usage is True
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
setattr(usage, "cost", logging_obj._response_cost_calculator(result=response))
_stamp_streaming_usage_cost(usage, response, logging_obj)
_set_stream_builder_response_cost(response, logging_obj)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)

173
litellm/model_prices_and_context_window_backup.json
View file

@ -23514,6 +23514,63 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_batches": 3.75e-07,
"input_cost_per_token_flex": 3.75e-07,
"litellm_provider": "vertex_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 3.75e-06,
"output_cost_per_token": 3.75e-06,
"output_cost_per_token_batches": 1.875e-06,
"output_cost_per_token_flex": 1.875e-06,
"regional_endpoint_uplift_multiplier": 1.1,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
@ -25351,6 +25408,65 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"gemini/gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_batches": 3.75e-07,
"input_cost_per_token_flex": 3.75e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 3.75e-06,
"output_cost_per_token": 3.75e-06,
"output_cost_per_token_batches": 1.875e-06,
"output_cost_per_token_flex": 1.875e-06,
"rpm": 2000,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_output": false,
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 800000,
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"gemini/gemini-omni-flash-preview": {
"input_cost_per_audio_token": 1.5e-06,
"input_cost_per_token": 1.5e-06,
@ -25759,6 +25875,63 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_batches": 3.75e-07,
"input_cost_per_token_flex": 3.75e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 3.75e-06,
"output_cost_per_token": 3.75e-06,
"output_cost_per_token_batches": 1.875e-06,
"output_cost_per_token_flex": 1.875e-06,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_output": false,
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"gemini/gemini-2.5-pro-preview-tts": {
"cache_read_input_token_cost": 1.25e-07,
"input_cost_per_audio_token": 7e-07,

25
litellm/proxy/common_utils/model_listing_utils.py
View file

@ -10,13 +10,36 @@ legacy internal names with `general_settings.use_team_public_model_name: false`.
from __future__ import annotations
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, cast
if TYPE_CHECKING:
from litellm.router import Router
def configured_display_names(
entries: Sequence[tuple[str, str]],
llm_router: Router | None,
) -> Mapping[str, str]:
"""response_id -> configured `model_info.display_name` for the listing entries
that have one.
Metadata is looked up by each entry's internal lookup id (so team-scoped rows
resolve), while the returned map is keyed by the public response id the
Anthropic-shaped listing is built from. Entries without a configured name are
omitted so the listing falls back to the id itself.
"""
if llm_router is None:
return MappingProxyType({})
resolved: Final = (
(response_id, llm_router.get_configured_display_name(lookup_id)) for response_id, lookup_id in entries
)
return MappingProxyType(
{response_id: display_name for response_id, display_name in resolved if display_name is not None}
)
class TeamModelNameTranslator:
"""Translates internal team routing keys to their public names for the model
listing/retrieve responses. Stateless; the live router and general_settings

21
litellm/proxy/proxy_server.py
View file

@ -352,7 +352,10 @@ from litellm.proxy.common_utils.load_config_utils import (
get_file_contents_from_s3,
)
from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations
from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator
from litellm.proxy.common_utils.model_listing_utils import (
TeamModelNameTranslator,
configured_display_names,
)
from litellm.proxy.common_utils.openai_endpoint_utils import (
remove_sensitive_info_from_deployment,
)
@ -10223,7 +10226,8 @@ async def model_list(
# The internal routing key drives the metadata/fallback lookup, while the
# public name is what the client sees as the model id.
model_data = []
for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings):
admin_entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings)
for response_id, lookup_id in admin_entries:
model_info = create_model_info_response(
model_id=lookup_id,
provider="openai",
@ -10236,7 +10240,10 @@ async def model_list(
if wants_anthropic_format:
admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
return create_anthropic_model_list_response(admin_listing)
return create_anthropic_model_list_response(
admin_listing,
display_names=configured_display_names(admin_entries, llm_router),
)
return dict(
data=model_data,
@ -10267,7 +10274,8 @@ async def model_list(
# The internal routing key drives the metadata/fallback lookup, while the
# public name is what the client sees as the model id.
model_data = []
for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings):
entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings)
for response_id, lookup_id in entries:
model_info = create_model_info_response(
model_id=lookup_id,
provider="openai",
@ -10280,7 +10288,10 @@ async def model_list(
if wants_anthropic_format:
listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
return create_anthropic_model_list_response(listing)
return create_anthropic_model_list_response(
listing,
display_names=configured_display_names(entries, llm_router),
)
return dict(
data=model_data,

29
litellm/rerank_api/main.py
View file

@ -6,6 +6,7 @@ from typing import Any, Final, Literal
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler
@ -43,10 +44,23 @@ async def arerank(
"""
Async: Reranks a list of documents based on their relevance to the query
"""
_custom_llm_provider: str | None = (
None # rebind-ok: set by the declared-provider guard or the get_llm_provider unpack; read in the except
)
try:
loop: Final = asyncio.get_event_loop()
kwargs["arerank"] = True
declared_provider: Final = declared_authenticating_provider(model, custom_llm_provider)
if declared_provider is not None:
_custom_llm_provider = declared_provider # rebind-ok: see pre-declaration above
else:
_, _custom_llm_provider, _, _ = litellm.get_llm_provider( # rebind-ok: see pre-declaration above
model=model,
custom_llm_provider=custom_llm_provider,
api_base=kwargs.get("api_base", None),
)
func: Final = partial(
rerank,
model,
@ -70,7 +84,11 @@ async def arerank(
response = init_response
return response
except Exception as e:
raise e
raise exception_type(
model=model,
custom_llm_provider=_custom_llm_provider or custom_llm_provider,
original_exception=e,
)
@client
@ -115,6 +133,7 @@ def rerank(
model_info: Final = kwargs.get("model_info", None)
user: Final = kwargs.get("user", None)
client: Final = kwargs.get("client", None)
_custom_llm_provider: str | None = None # rebind-ok: set by the get_llm_provider unpack; read in the except
try:
_is_async: Final = kwargs.pop("arerank", False) is True
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -127,7 +146,7 @@ def rerank(
(
model,
_custom_llm_provider,
_custom_llm_provider, # rebind-ok: see pre-declaration above
dynamic_api_key,
dynamic_api_base,
) = litellm.get_llm_provider(
@ -538,4 +557,8 @@ def rerank(
return response
except Exception as e:
verbose_logger.error("Error in rerank: %s", e)
raise exception_type(model=model, custom_llm_provider=custom_llm_provider, original_exception=e)
raise exception_type(
model=model,
custom_llm_provider=_custom_llm_provider or custom_llm_provider,
original_exception=e,
)

10
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -1169,16 +1169,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
def _emit_response_completed_event(self, litellm_model_response: ModelResponse) -> ResponseCompletedEvent | None:
if litellm_model_response:
# Add cost to usage object if include_cost_in_streaming_usage is True
if litellm.include_cost_in_streaming_usage and self.litellm_logging_obj is not None:
usage: Final[object] = getattr(litellm_model_response, "usage", None)
if usage is not None:
setattr(
usage,
"cost",
self.litellm_logging_obj._response_cost_calculator(result=litellm_model_response),
)
# Transform the response
responses_api_response: Final = (
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(

46
litellm/responses/streaming_iterator.py
View file

@ -407,23 +407,7 @@ class BaseResponsesAPIStreamingIterator:
openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED,
):
self.completed_response = openai_responses_api_chunk
# Add cost to usage object if include_cost_in_streaming_usage is True
if litellm.include_cost_in_streaming_usage and self.logging_obj is not None:
response_obj: Final[ResponsesAPIResponse | None] = getattr(
openai_responses_api_chunk, "response", None
)
if response_obj:
usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None)
if usage_obj is not None:
try:
cost: Final[float | None] = self.logging_obj._response_cost_calculator(
result=response_obj
)
if cost is not None:
setattr(usage_obj, "cost", cost)
except Exception:
# Best-effort usage cost annotation should not break stream replay.
pass
_stamp_responses_usage_cost(getattr(openai_responses_api_chunk, "response", None), self.logging_obj)
if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED:
self._handle_logging_failed_response()
@ -1274,6 +1258,24 @@ def _add_text_like_part_events(
)
def _stamp_responses_usage_cost(
response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None
) -> None:
if response_obj is None or logging_obj is None:
return
usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None)
if usage_obj is None:
return
if isinstance(getattr(usage_obj, "cost", None), (int, float)):
return
try:
cost: Final[float | None] = logging_obj._response_cost_calculator(result=response_obj)
except Exception:
return
if isinstance(cost, (int, float)) and cost > 0:
setattr(usage_obj, "cost", cost)
def build_synthetic_response_events(
*,
transformed: ResponsesAPIResponse,
@ -1281,15 +1283,7 @@ def build_synthetic_response_events(
chunk_size: int,
) -> list[ResponsesAPIStreamingResponse]:
openai_types: Final = _get_openai_response_types()
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
usage_obj: Final = transformed.usage if hasattr(transformed, "usage") else None
if usage_obj is not None:
try:
cost: Final[float | None] = logging_obj._response_cost_calculator(result=transformed)
if cost is not None:
setattr(usage_obj, "cost", cost)
except Exception:
pass
_stamp_responses_usage_cost(transformed, logging_obj)
events: Final[list[ResponsesAPIStreamingResponse]] = [
_build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed),

20
litellm/router.py
View file

@ -9746,6 +9746,26 @@ class Router:
coerce_token_limit(model_info.get("max_output_tokens")),
)
def get_configured_display_name(self, model_name: str) -> "str | None":
"""
Return the display_name explicitly configured in a concrete deployment's
model_info for model_name, via O(1) index lookup.
Returns None for wildcard-expanded or unknown names, and treats a
non-string or empty configured value as absent rather than failing the
listing. Like get_configured_token_limits, this never triggers pattern
matching or deep copies, so it is safe to call per listed model on the
/v1/models hot path.
"""
deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name)
if deployment is None:
return None
display_name: Final = deployment.model_info.get("display_name")
if isinstance(display_name, str) and display_name.strip():
return display_name
return None
def get_deployment_credentials_with_provider(
self, model_id: str, team_id: str | None = None
) -> dict[str, Any] | None:

173
model_prices_and_context_window.json
View file

@ -23514,6 +23514,63 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_batches": 3.75e-07,
"input_cost_per_token_flex": 3.75e-07,
"litellm_provider": "vertex_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 3.75e-06,
"output_cost_per_token": 3.75e-06,
"output_cost_per_token_batches": 1.875e-06,
"output_cost_per_token_flex": 1.875e-06,
"regional_endpoint_uplift_multiplier": 1.1,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 2e-07,
@ -25351,6 +25408,65 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"gemini/gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_batches": 3.75e-07,
"input_cost_per_token_flex": 3.75e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 3.75e-06,
"output_cost_per_token": 3.75e-06,
"output_cost_per_token_batches": 1.875e-06,
"output_cost_per_token_flex": 1.875e-06,
"rpm": 2000,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_output": false,
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 800000,
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"gemini/gemini-omni-flash-preview": {
"input_cost_per_audio_token": 1.5e-06,
"input_cost_per_token": 1.5e-06,
@ -25759,6 +25875,63 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_batches": 3.75e-07,
"input_cost_per_token_flex": 3.75e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 3.75e-06,
"output_cost_per_token": 3.75e-06,
"output_cost_per_token_batches": 1.875e-06,
"output_cost_per_token_flex": 1.875e-06,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_output": false,
"supports_audio_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
"gemini/gemini-2.5-pro-preview-tts": {
"cache_read_input_token_cost": 1.25e-07,
"input_cost_per_audio_token": 7e-07,

45
tests/e2e/test_junit_properties.py
View file

@ -24,25 +24,10 @@ from junit_properties import (
)
class FakeMarker:
def __init__(self, name: str, *args: object) -> None:
self.name = name
self.args = args
class FakeItem:
"""The three attributes junit_properties reads off a pytest Item."""
def __init__(
self, nodeid: str, location: tuple[str, int | None, str], markers: tuple[FakeMarker, ...] = ()
) -> None:
self.nodeid = nodeid
self.location = location
self.user_properties: list[tuple[str, str]] = []
self._markers = markers
def iter_markers(self, name: str):
return (marker for marker in self._markers if marker.name == name)
def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item:
"""The Item pytest collected for test ``name`` in this file: the real nodeid,
location and marker machinery the collection hook reads, as pytest built it."""
return next(item for item in request.session.items if item.path == request.path and item.name == name)
def repo_root() -> Path | None:
@ -109,22 +94,22 @@ class TestSourceFromLocation:
class TestResultProperties:
def test_every_test_carries_package_covers_and_source(self) -> None:
item = FakeItem(
"logging/test_x.py::TestFoo::test_bar",
("logging/test_x.py", 40, "TestFoo.test_bar"),
(FakeMarker("covers", "LOG-1", "LOG-2"),),
)
assert result_properties(item) == (
("package", "logging"),
def test_every_test_carries_package_covers_and_source(self, request: pytest.FixtureRequest) -> None:
"""Read off this test's own collected Item, so the nodeid and location are
whatever pytest reports for the launch shape in use, and the marker is added
at run time so the coverage registry's collect-only pass never sees it."""
test = type(self).test_every_test_carries_package_covers_and_source
request.applymarker(pytest.mark.covers("LOG-1", "LOG-2"))
assert result_properties(collected_item(request, test.__name__)) == (
("package", "root"),
("covers", "LOG-1,LOG-2"),
("source", "tests/e2e/logging/test_x.py:41"),
("source", f"tests/e2e/test_junit_properties.py:{test.__code__.co_firstlineno}"),
)
def test_attach_is_idempotent(self) -> None:
def test_attach_is_idempotent(self, request: pytest.FixtureRequest) -> None:
"""Collection can run the hook more than once; a second pass must not
double the <property> entries in the report."""
item = FakeItem("logging/test_x.py::test_bar", ("logging/test_x.py", 40, "test_bar"))
item = collected_item(request, type(self).test_attach_is_idempotent.__name__)
attach_result_properties(item)
attach_result_properties(item)
assert [name for name, _ in item.user_properties] == ["package", "covers", "source"]

20
tests/ocr_tests/test_ocr_vertex_ai.py
View file

@ -5,9 +5,11 @@ Note: Vertex AI OCR automatically converts URLs to base64 data URIs since
the Vertex AI endpoint doesn't have internet access.
"""
import os
import json
import os
import tempfile
from typing import Final
import pytest
from base_ocr_unit_tests import BaseOCRTest
@ -139,3 +141,19 @@ def test_vertex_ai_ocr_routing():
assert isinstance(
deepseek_variant, VertexAIDeepSeekOCRConfig
), "DeepSeek variant should route to VertexAIDeepSeekOCRConfig"
@pytest.mark.parametrize("model", ("deepseek-ocr-maas", "deepseek-ai/deepseek-ocr-maas"))
def test_deepseek_request_uses_single_provider_namespace(model: str) -> None:
from litellm.llms.vertex_ai.ocr.deepseek_transformation import (
VertexAIDeepSeekOCRConfig,
)
request: Final = VertexAIDeepSeekOCRConfig().transform_ocr_request(
model=model,
document={"type": "image_url", "image_url": "data:image/png;base64,AA=="},
optional_params={},
headers={},
)
assert request.data["model"] == "deepseek-ai/deepseek-ocr-maas"

199
tests/test_litellm/integrations/test_custom_guardrail.py
View file

@ -2237,3 +2237,202 @@ class TestRecordsOwnGuardrailInformation:
)
assert _guardrail_entries(request_data) == []
class _ApplyOnlyObserver(CustomGuardrail):
"""Overrides only apply_guardrail, like panw_prisma_airs; inherits async_logging_hook."""
def __init__(self, block: bool = False):
from litellm.types.guardrails import GuardrailEventHooks
super().__init__(guardrail_name="apply-only-observer", event_hook=GuardrailEventHooks.logging_only)
self.block = block
self.calls: list = []
@log_guardrail_information
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
from fastapi import HTTPException
self.calls.append((input_type, list(inputs.get("texts") or [])))
if self.block:
raise HTTPException(status_code=400, detail={"error": "flagged"})
return GenericGuardrailAPIInputs(texts=["[MASKED]" for _ in inputs.get("texts") or []])
def _logged_call(messages: list | str) -> tuple[dict, object]:
from litellm.types.utils import Choices, Message, ModelResponse
response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="general kenobi"))])
kwargs = {
"model": "gpt-5.4-mini",
"messages": messages,
"litellm_call_id": "call-1",
"litellm_params": {"metadata": {"user_api_key_user_id": "u1"}},
"optional_params": {},
"standard_logging_object": {"guardrail_information": None},
}
return kwargs, response
class TestLoggingOnlyApplyGuardrail:
"""LIT-4876 regression: a guardrail in mode logging_only that implements only
apply_guardrail must still run against the logged request and response and
record guardrail_information, instead of inheriting the CustomLogger no-op."""
@pytest.mark.asyncio
async def test_runs_apply_guardrail_observe_only_and_records_verdict(self):
guardrail = _ApplyOnlyObserver()
messages = [{"role": "user", "content": "hello there"}]
kwargs, response = _logged_call(messages)
out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value)
assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])]
assert out_kwargs["messages"] == [{"role": "user", "content": "hello there"}]
assert out_response.choices[0].message.content == "general kenobi"
entries = out_kwargs["standard_logging_object"]["guardrail_information"]
assert [e["guardrail_name"] for e in entries] == ["apply-only-observer", "apply-only-observer"]
assert {e["guardrail_mode"] for e in entries} == {"logging_only"}
assert {e["guardrail_status"] for e in entries} == {"success"}
assert "standard_logging_guardrail_information" not in kwargs["litellm_params"]["metadata"]
assert kwargs["standard_logging_object"] == {"guardrail_information": None}
@pytest.mark.asyncio
async def test_appends_to_pre_call_verdicts_without_duplicating_them(self):
guardrail = _ApplyOnlyObserver()
kwargs, response = _logged_call([{"role": "user", "content": "hello there"}])
pre_call_entry = {"guardrail_name": "pii-blocker", "guardrail_mode": "pre_call", "guardrail_status": "success"}
kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] = [pre_call_entry]
kwargs["standard_logging_object"]["guardrail_information"] = [pre_call_entry]
out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value)
entries = out_kwargs["standard_logging_object"]["guardrail_information"]
assert [e["guardrail_name"] for e in entries] == ["pii-blocker", "apply-only-observer", "apply-only-observer"]
assert kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] == [pre_call_entry]
@pytest.mark.asyncio
async def test_request_copy_failure_is_swallowed(self):
import threading
guardrail = _ApplyOnlyObserver()
kwargs, response = _logged_call([{"role": "user", "content": "hello there", "lock": threading.Lock()}])
out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value)
assert guardrail.calls == []
assert out_kwargs is kwargs
assert out_response is response
@pytest.mark.asyncio
async def test_block_verdict_is_recorded_without_raising(self):
guardrail = _ApplyOnlyObserver(block=True)
kwargs, response = _logged_call([{"role": "user", "content": "flagged content"}])
out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value)
assert guardrail.calls == [("request", ["flagged content"])]
entries = out_kwargs["standard_logging_object"]["guardrail_information"]
assert [e["guardrail_status"] for e in entries] == ["guardrail_intervened"]
@pytest.mark.asyncio
async def test_call_type_without_translation_is_skipped(self):
guardrail = _ApplyOnlyObserver()
kwargs, response = _logged_call([{"role": "user", "content": "hello there"}])
out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.amoderation.value)
assert guardrail.calls == []
assert out_kwargs["standard_logging_object"]["guardrail_information"] is None
@pytest.mark.asyncio
async def test_aembedding_scans_logged_input(self):
from litellm.types.utils import EmbeddingResponse
guardrail = _ApplyOnlyObserver()
kwargs, _ = _logged_call("hello there")
response = EmbeddingResponse(data=[{"embedding": [0.1], "index": 0, "object": "embedding"}])
out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.aembedding.value)
assert guardrail.calls == [("request", ["hello there"])]
assert out_kwargs["messages"] == "hello there"
assert out_response is response
entries = out_kwargs["standard_logging_object"]["guardrail_information"]
assert [e["guardrail_status"] for e in entries] == ["success"]
@pytest.mark.asyncio
async def test_native_lifecycle_hook_guardrail_is_left_alone(self):
class _NativeHooks(_ApplyOnlyObserver):
use_native_lifecycle_hooks = True
guardrail = _NativeHooks()
kwargs, response = _logged_call([{"role": "user", "content": "hello there"}])
out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value)
assert guardrail.calls == []
assert out_kwargs is kwargs
assert out_response is response
@pytest.mark.asyncio
async def test_aresponses_scans_logged_messages_when_input_is_cleared(self):
from litellm.types.llms.openai import ResponsesAPIResponse
guardrail = _ApplyOnlyObserver()
kwargs, _ = _logged_call([{"role": "user", "content": "hello there"}])
kwargs["input"] = None
response = ResponsesAPIResponse(
id="resp_1",
created_at=1,
model="gpt-5.4-mini",
object="response",
status="completed",
output=[
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "general kenobi"}],
}
],
)
out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.aresponses.value)
assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])]
entries = out_kwargs["standard_logging_object"]["guardrail_information"]
assert [e["guardrail_status"] for e in entries] == ["success", "success"]
@pytest.mark.asyncio
async def test_async_success_handler_records_verdict_in_standard_logging_object(self):
import datetime as dt
from litellm.litellm_core_utils.litellm_logging import Logging
guardrail = _ApplyOnlyObserver()
guardrail.default_on = True
messages = [{"role": "user", "content": "hello there"}]
_, response = _logged_call(messages)
logging_obj = Logging(
model="gpt-5.4-mini",
messages=messages,
stream=False,
call_type=CallTypes.acompletion.value,
start_time=dt.datetime.now(),
litellm_call_id="call-1",
function_id="fn-1",
dynamic_async_success_callbacks=[guardrail],
)
logging_obj.update_environment_variables(
litellm_params={"metadata": {}}, optional_params={}, model="gpt-5.4-mini", custom_llm_provider="openai"
)
await logging_obj.async_success_handler(
result=response, start_time=dt.datetime.now(), end_time=dt.datetime.now()
)
assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])]
entries = logging_obj.model_call_details["standard_logging_object"]["guardrail_information"]
assert [e["guardrail_status"] for e in entries] == ["success", "success"]

80
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -4200,6 +4200,86 @@ def test_generic_cost_per_token_gemini_37_flash(_local_model_cost_map):
assert completion_cost == pytest.approx(0.001875)
GEMINI_38_FLASH_LAUNCH_PRICING = [
("gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08),
("gemini/gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08),
("vertex_ai/gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08),
]
@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_38_FLASH_LAUNCH_PRICING)
def test_gemini_38_flash_launch_pricing(model, input_cost, output_cost, cache_read_cost, _local_model_cost_map):
model_cost_map = litellm.model_cost[model]
assert model_cost_map["input_cost_per_token"] == input_cost
assert model_cost_map["output_cost_per_token"] == output_cost
assert model_cost_map["output_cost_per_reasoning_token"] == output_cost
assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost
assert model_cost_map["mode"] == "chat"
assert model_cost_map["supports_reasoning"] is True
assert model_cost_map["supports_function_calling"] is True
assert model_cost_map["max_input_tokens"] == 1048576
GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH = (
"input_cost_per_token",
"output_cost_per_token",
"output_cost_per_reasoning_token",
"cache_read_input_token_cost",
"input_cost_per_token_batches",
"output_cost_per_token_batches",
"input_cost_per_token_flex",
"output_cost_per_token_flex",
"cache_read_input_token_cost_flex",
"input_cost_per_token_priority",
"output_cost_per_token_priority",
"cache_read_input_token_cost_priority",
"search_context_cost_per_query",
"google_maps_grounding_cost_per_query",
"prompt_cache_min_tokens",
"max_input_tokens",
"max_output_tokens",
"supports_reasoning",
"supports_function_calling",
"supports_prompt_caching",
"supports_vision",
"supports_pdf_input",
"supports_audio_input",
"supports_video_input",
"supports_response_schema",
"supports_tool_choice",
"supports_web_search",
"supports_url_context",
)
@pytest.mark.parametrize("prefix", ["", "gemini/", "vertex_ai/"])
def test_gemini_38_flash_matches_37_flash_promotional_pricing(prefix, _local_model_cost_map):
new_model = litellm.model_cost[f"{prefix}gemini-3.8-flash"]
old_model = litellm.model_cost[f"{prefix}gemini-3.7-flash"]
for field in GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH:
assert new_model[field] == old_model[field], field
def test_generic_cost_per_token_gemini_38_flash(_local_model_cost_map):
usage = Usage(
prompt_tokens=1000,
completion_tokens=500,
total_tokens=1500,
completion_tokens_details=CompletionTokensDetailsWrapper(
reasoning_tokens=200,
text_tokens=300,
),
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=1000),
)
prompt_cost, completion_cost = generic_cost_per_token(
model="gemini-3.8-flash",
usage=usage,
custom_llm_provider="gemini",
)
assert prompt_cost == pytest.approx(0.00075)
assert completion_cost == pytest.approx(0.001875)
def test_grok_46_launch_pricing(_local_model_cost_map):
model_cost_map = litellm.model_cost["xai/grok-4.6"]
assert model_cost_map["input_cost_per_token"] == 2e-06

53
tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py
View file

@ -592,6 +592,59 @@ def test_stream_chunk_builder_litellm_usage_chunks():
assert usage.total_tokens == 77
def test_calculate_usage_honors_openai_sdk_completion_usage_chunks():
from openai.types.completion_usage import CompletionUsage
content_chunk = ModelResponseStream(
id="chatcmpl-sdk-usage-1",
created=1745513206,
model="mantle-claude",
object="chat.completion.chunk",
system_fingerprint=None,
choices=[
StreamingChoices(
finish_reason="stop",
index=0,
delta=Delta(
provider_specific_fields=None,
content="ok",
role=None,
function_call=None,
tool_calls=None,
audio=None,
),
logprobs=None,
)
],
provider_specific_fields=None,
stream_options={"include_usage": True},
)
usage_chunk = ModelResponseStream(
id="chatcmpl-sdk-usage-1",
created=1745513207,
model="mantle-claude",
object="chat.completion.chunk",
system_fingerprint=None,
choices=[],
provider_specific_fields=None,
stream_options={"include_usage": True},
)
usage_chunk.usage = CompletionUsage(
prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704
)
assert type(usage_chunk.usage) is CompletionUsage
chunks = [content_chunk, usage_chunk]
usage = ChunkProcessor(chunks=chunks).calculate_usage(
chunks=chunks, model="mantle-claude", completion_output=""
)
assert usage.prompt_tokens == 20
assert usage.completion_tokens == 60
assert usage.total_tokens == 80
assert getattr(usage, "cost", None) == pytest.approx(0.000704)
def test_get_model_from_chunks_azure_model_router():
"""
Test that _get_model_from_chunks finds the actual model from Azure Model Router chunks.

3
tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py
View file

@ -1096,10 +1096,13 @@ def test_natively_signed_parallel_turn_never_carries_a_placeholder(model):
"gemini-3.5-flash",
"gemini-3.6-flash",
"gemini-3.7-flash",
"gemini-3.8-flash",
"vertex_ai/gemini-3.5-flash",
"vertex_ai/gemini-3.7-flash",
"vertex_ai/gemini-3.8-flash",
"gemini/gemini-3.5-flash",
"gemini/gemini-3.7-flash",
"gemini/gemini-3.8-flash",
],
)
def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model):

12
tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
View file

@ -1185,6 +1185,18 @@ def test_vertex_ai_map_thinking_param_with_budget_tokens_0():
}
def test_vertex_ai_map_thinking_param_without_budget_tokens_for_gemini_3():
v = VertexGeminiConfig()
result = v.map_openai_params(
non_default_params={"thinking": {"type": "enabled"}},
optional_params={},
model="gemini-3.5-flash",
drop_params=False,
)
assert result["thinkingConfig"] == {"includeThoughts": True}
def test_vertex_ai_map_tools():
v = VertexGeminiConfig()
optional_params = {}

78
tests/test_litellm/proxy/proxy_server/test_routes_models.py
View file

@ -45,6 +45,7 @@ def patched_models(monkeypatch):
deployment = MagicMock()
deployment.litellm_params.model = "gpt-4"
router.get_deployment_by_model_group_name = MagicMock(return_value=deployment)
router.get_configured_display_name = MagicMock(return_value=None)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
@ -187,6 +188,83 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as
assert (claude["max_input_tokens"], claude["max_tokens"]) == (500000, 4096)
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
def test_anthropic_format_uses_configured_display_name(client, auth_as, patched_models, path):
"""A deployment's ``model_info.display_name`` becomes the Anthropic-native
``display_name`` so Claude Code's picker shows a clean name while the id keeps
routing; models without one keep the id fallback, and the OpenAI-shaped
listing carries no display_name either way."""
def _configured(model_name):
return "Kimi K3" if model_name == "gpt-4" else None
patched_models.get_configured_display_name = MagicMock(side_effect=_configured)
with auth_as():
anthropic_response = client.get(path, headers={"anthropic-version": "2023-06-01"})
openai_response = client.get(path)
assert anthropic_response.status_code == 200
gpt_4, claude = anthropic_response.json()["data"]
assert (gpt_4["id"], gpt_4["display_name"]) == ("gpt-4", "Kimi K3")
assert (claude["id"], claude["display_name"]) == ("claude-sonnet", "claude-sonnet")
assert openai_response.status_code == 200
openai_models = openai_response.json()["data"]
assert [m["id"] for m in openai_models] == ["gpt-4", "claude-sonnet"]
assert all("display_name" not in m for m in openai_models)
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
def test_anthropic_display_name_resolved_via_internal_team_key(
client, auth_as, patched_models, monkeypatch, params
):
"""For a team-scoped row the configured display name must be looked up by the
internal routing key while the entry itself is keyed by the public name, so
the clean name lands on the id the client actually sees."""
from litellm.proxy import utils as proxy_utils
from litellm.proxy.auth import model_checks
internal_name = "model_name_team-1_c0ffee"
patched_models.get_model_list = MagicMock(
return_value=[
{
"model_name": internal_name,
"model_info": {
"team_id": "team-1",
"team_public_model_name": "gpt-4-team",
},
}
]
)
patched_models.get_model_names = MagicMock(return_value=[internal_name])
patched_models.get_configured_display_name = MagicMock(
side_effect=lambda model_name: "Team GPT" if model_name == internal_name else None
)
async def _fake_get_available_models_for_user(**kwargs):
return [internal_name]
monkeypatch.setattr(
proxy_utils,
"get_available_models_for_user",
_fake_get_available_models_for_user,
)
monkeypatch.setattr(
model_checks, "get_complete_model_list", lambda **kwargs: [internal_name]
)
with auth_as():
response = client.get(
"/v1/models", params=params, headers={"anthropic-version": "2023-06-01"}
)
assert response.status_code == 200
(entry,) = response.json()["data"]
assert (entry["id"], entry["display_name"]) == ("gpt-4-team", "Team GPT")
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
def test_get_models_invalid_scope_returns_400(client, auth_as, patched_models, path):
"""Pins: ``GET /v1/models``, ``GET /models`` (error path: invalid scope)."""

26
tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py
View file

@ -19,7 +19,10 @@ from litellm.proxy._types import (
LitellmUserRoles,
UserAPIKeyAuth,
)
from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator
from litellm.proxy.common_utils.model_listing_utils import (
TeamModelNameTranslator,
configured_display_names,
)
from litellm.proxy.proxy_server import (
_get_proxy_model_info,
_translate_model_name_for_response,
@ -1391,6 +1394,27 @@ def test_resolve_public_name_respects_legacy_flag():
)
def test_configured_display_names_keyed_by_response_id():
"""The map is keyed by the public response id while the router lookup uses
the internal routing key, and entries without a configured name are omitted."""
router = MagicMock()
router.get_configured_display_name = MagicMock(
side_effect=lambda model_name: "Team Sonnet" if model_name == "model_name_team-abc-123_4a6b8" else None
)
assert configured_display_names(
entries=[
("team-claude-sonnet", "model_name_team-abc-123_4a6b8"),
("gpt-4o", "gpt-4o"),
],
llm_router=router,
) == {"team-claude-sonnet": "Team Sonnet"}
def test_configured_display_names_empty_without_router():
assert configured_display_names(entries=[("gpt-4o", "gpt-4o")], llm_router=None) == {}
@pytest.mark.asyncio
async def test_retrieve_model_by_public_name_returns_200(monkeypatch):
"""Regression: `GET /v1/models/{public_name}` must NOT 404. The listing

93
tests/test_litellm/rerank_api/test_main.py
View file

@ -111,6 +111,99 @@ def test_together_rerank_honors_api_base(respx_mock: respx.MockRouter):
assert mock_route.calls[0].request.headers["authorization"] == "Bearer fake-together-key"
DASHSCOPE_404_BODY = {
"error": {
"message": "The model `does-not-exist` does not exist or you do not have access to it.",
"type": "invalid_request_error",
"param": None,
"code": "model_not_found",
},
"request_id": "mock-request-id",
}
def test_rerank_error_names_provider_and_keeps_body(respx_mock: respx.MockRouter, monkeypatch):
"""Regression for the rerank error path mapping with the unresolved provider param:
a provider 404 surfaced as 'None - ' instead of naming the provider and its error body."""
monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False)
monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False)
mock_route = respx_mock.post("https://dashscope.example/v1/reranks")
mock_route.return_value = httpx.Response(404, json=DASHSCOPE_404_BODY)
with pytest.raises(litellm.NotFoundError) as exc_info:
litellm.rerank(
model="dashscope/does-not-exist",
query=MARKER_QUERY,
documents=[MARKER_DOC],
api_key="fake-dashscope-key",
api_base="https://dashscope.example/v1",
)
assert mock_route.called
assert "DashscopeException" in str(exc_info.value)
assert "does not exist or you do not have access to it" in str(exc_info.value)
assert "None - " not in str(exc_info.value)
@pytest.mark.asyncio
async def test_arerank_error_is_mapped_to_litellm_exception(respx_mock: respx.MockRouter, monkeypatch):
"""Regression for arerank's bare re-raise: provider errors escaped as raw
provider exception classes instead of the mapped litellm exception contract."""
monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False)
monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False)
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
mock_route = respx_mock.post("https://dashscope.example/v1/reranks")
mock_route.return_value = httpx.Response(404, json=DASHSCOPE_404_BODY)
with pytest.raises(litellm.NotFoundError) as exc_info:
await litellm.arerank(
model="dashscope/does-not-exist",
query=MARKER_QUERY,
documents=[MARKER_DOC],
api_key="fake-dashscope-key",
api_base="https://dashscope.example/v1",
)
assert mock_route.called
assert "DashscopeException" in str(exc_info.value)
assert "does not exist or you do not have access to it" in str(exc_info.value)
assert "None - " not in str(exc_info.value)
@pytest.mark.asyncio
async def test_arerank_declared_authenticating_provider_skips_resolution(monkeypatch):
"""Regression for the event-loop hazard in arerank's provider pre-resolution:
get_llm_provider runs the blocking OAuth device flow for github_copilot/chatgpt,
so arerank must adopt the declared provider instead of resolving it, while the
except path still maps with that declared provider."""
from litellm.llms.base_llm.chat.transformation import BaseLLMException
resolution_calls = []
def record_resolution(*args, **kwargs):
resolution_calls.append((args, kwargs))
return "gpt-4o", "github_copilot", None, None
def rerank_raises_provider_error(*args, **kwargs):
raise BaseLLMException(status_code=401, message='{"error":"bad key"}')
monkeypatch.setattr(litellm, "get_llm_provider", record_resolution)
monkeypatch.setattr("litellm.rerank_api.main.rerank", rerank_raises_provider_error)
with pytest.raises(litellm.AuthenticationError) as exc_info:
await litellm.arerank(
model="github_copilot/gpt-4o",
query=MARKER_QUERY,
documents=[MARKER_DOC],
)
assert resolution_calls == []
assert "Github_copilotException" in str(exc_info.value)
assert "None - " not in str(exc_info.value)
@pytest.mark.asyncio
async def test_together_rerank_async_honors_env_api_base(respx_mock: respx.MockRouter, monkeypatch):
"""Regression: TOGETHER_AI_API_BASE was honored by chat but ignored by rerank."""

52
tests/test_litellm/responses/test_streaming_iterator.py
View file

@ -326,3 +326,55 @@ def test_run_post_success_hooks_does_not_report_generation_time_as_overhead():
assert iterator.completed_response._hidden_params["_response_ms"] == 10000.0
assert "litellm_overhead_time_ms" not in iterator.completed_response._hidden_params
def _responses_api_response_with_usage() -> ResponsesAPIResponse:
from litellm.types.llms.openai import ResponseAPIUsage
return ResponsesAPIResponse(
id="resp_lit6427",
created_at=int(datetime(2025, 1, 1).timestamp()),
status="completed",
model="mantle-claude",
object="response",
output=[],
usage=ResponseAPIUsage(input_tokens=20, output_tokens=60, total_tokens=80),
)
def test_stamp_responses_usage_cost_stamps_computed_cost():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
response = _responses_api_response_with_usage()
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj._response_cost_calculator.return_value = 0.000704
_stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.000704)
logging_obj._response_cost_calculator.assert_called_once_with(result=response)
def test_stamp_responses_usage_cost_keeps_provider_reported_cost():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
response = _responses_api_response_with_usage()
setattr(response.usage, "cost", 0.5)
logging_obj = Mock(spec=LiteLLMLoggingObj)
_stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.5)
logging_obj._response_cost_calculator.assert_not_called()
def test_stamp_responses_usage_cost_survives_calculator_failure():
from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
response = _responses_api_response_with_usage()
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj._response_cost_calculator.side_effect = RuntimeError("cost map unavailable")
_stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) is None

66
tests/test_litellm/test_main.py
View file

@ -3150,8 +3150,8 @@ def _stream_builder_logging_obj() -> LiteLLMLogging:
return logging_obj
def test_stream_chunk_builder_reports_streaming_usage_cost_when_enabled(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False)
chunks: Final = [
_stream_builder_text_chunk("gpt-4o", "Hello "),
_stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"),
@ -3168,11 +3168,45 @@ def test_stream_chunk_builder_reports_streaming_usage_cost_when_enabled(monkeypa
assert response._hidden_params["response_cost"] == pytest.approx(usage_cost)
def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False)
def test_stream_chunk_builder_skips_stamp_when_cost_is_unpriceable():
import time as time_module
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
logging_obj: Final = LiteLLMLogging(
model="us.anthropic.claude-opus-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="completion",
start_time=time_module.time(),
litellm_call_id="stream-builder-alias-unpriceable",
function_id="1",
)
logging_obj.model_call_details["custom_llm_provider"] = "bedrock"
logging_obj.optional_params = {}
usage_chunk: Final = _stream_builder_text_chunk("bedrock-claude-opus-5", "")
usage_chunk.usage = Usage(prompt_tokens=40, completion_tokens=5, total_tokens=45)
chunks: Final = [
_stream_builder_text_chunk("bedrock-claude-opus-5", "Hello ", finish_reason="stop"),
usage_chunk,
]
response: Final = litellm.stream_chunk_builder(
chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj
)
assert response is not None
assert getattr(response.usage, "cost", None) is None
assert response._hidden_params.get("response_cost") is None
def test_stream_chunk_builder_keeps_provider_reported_usage_cost():
usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "")
usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5)
chunks: Final = [
_stream_builder_text_chunk("gpt-4o", "Hello "),
_stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"),
usage_chunk,
]
response: Final = litellm.stream_chunk_builder(
@ -3180,4 +3214,26 @@ def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent(
)
assert response is not None
assert response._hidden_params.get("response_cost") is None
assert getattr(response.usage, "cost", None) == pytest.approx(0.5)
assert response._hidden_params["response_cost"] == pytest.approx(0.5)
def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk():
from openai.types.completion_usage import CompletionUsage
usage_chunk: Final = _stream_builder_text_chunk("mantle-claude", "")
usage_chunk.usage = CompletionUsage(prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704)
assert type(usage_chunk.usage) is CompletionUsage
chunks: Final = [
_stream_builder_text_chunk("mantle-claude", "Hello "),
_stream_builder_text_chunk("mantle-claude", "world.", finish_reason="stop"),
usage_chunk,
]
response: Final = litellm.stream_chunk_builder(chunks=chunks, messages=[{"role": "user", "content": "hi"}])
assert response is not None
assert response.usage.prompt_tokens == 20
assert response.usage.completion_tokens == 60
assert getattr(response.usage, "cost", None) == pytest.approx(0.000704)
assert response._hidden_params["response_cost"] == pytest.approx(0.000704)

65
tests/test_litellm/test_router.py
View file

@ -7271,6 +7271,71 @@ def test_get_configured_token_limits_coerces_numeric_strings():
assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000)
def test_get_configured_display_name_reads_deployment_model_info():
router = litellm.Router(
model_list=[
{
"model_name": "Kimi K3-claude-compatible",
"litellm_params": {"model": "openai/some-unmapped-model"},
"model_info": {"display_name": "Kimi K3"},
}
]
)
assert router.get_configured_display_name("Kimi K3-claude-compatible") == "Kimi K3"
def test_get_configured_display_name_returns_none_for_unset_or_unknown():
router = litellm.Router(
model_list=[
{
"model_name": "no-display-model",
"litellm_params": {"model": "openai/some-unmapped-model"},
}
]
)
assert router.get_configured_display_name("no-display-model") is None
assert router.get_configured_display_name("not-a-real-model") is None
def test_get_configured_display_name_skips_wildcard_pattern_matching():
router = litellm.Router(
model_list=[
{
"model_name": "bedrock/*",
"litellm_params": {"model": "bedrock/*"},
"model_info": {"display_name": "Bedrock"},
}
]
)
with patch.object(
router.pattern_router, "route", side_effect=AssertionError("pattern route called")
):
assert (
router.get_configured_display_name("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0")
is None
)
def test_get_configured_display_name_treats_malformed_values_as_absent():
malformed = ["", " ", 12345, ["Kimi K3"], {"name": "Kimi K3"}, True]
router = litellm.Router(
model_list=[
{
"model_name": f"bad-display-{i}",
"litellm_params": {"model": "openai/some-unmapped-model"},
"model_info": {"display_name": bad},
}
for i, bad in enumerate(malformed)
]
)
for i in range(len(malformed)):
assert router.get_configured_display_name(f"bad-display-{i}") is None
@pytest.mark.asyncio
async def test_acreate_batch_disable_fallbacks_surfaces_owning_provider_error():
router = litellm.Router(

1
tests/test_litellm/test_utils.py
View file

@ -4655,6 +4655,7 @@ GEMINI_4096_CACHE_MIN_MODELS: Final = tuple(
"gemini-3.5-flash",
"gemini-3.6-flash",
"gemini-3.7-flash",
"gemini-3.8-flash",
"gemini-3.1-pro-preview",
"gemini-3.1-pro-preview-customtools",
)