scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-16 23:41:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

Merge commit '79fc5153d3' into litellm_otel_gen_ai_system_none

Browse source 
# Conflicts:
#	litellm/integrations/opentelemetry.py
This commit is contained in:
mateo-berri 2026-09-16 13:38:40 -07:00
commit 4c79abfa0c
26 changed files with 1033 additions and 286 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
.github/workflows/test-rust.yml vendored
View file

@ -70,7 +70,7 @@ env:
jobs:
rust-lint:
runs-on: ubuntu-latest
timeout-minutes: 10
timeout-minutes: 15
defaults:
run:
working-directory: litellm-rust
@ -81,24 +81,48 @@ jobs:
- run: rustup toolchain install --no-self-update
- run: cargo fmt --check
- run: cargo fmt --all --check
- uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
path: |
~/.cargo/registry
~/.cargo/git
litellm-rust/target
key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-${{ github.job }}-
workspaces: litellm-rust
cache-on-failure: true
- run: cargo clippy --workspace --all-targets --locked -- -D warnings
rust-test:
runs-on: ubuntu-latest
timeout-minutes: 30
timeout-minutes: 20
defaults:
run:
working-directory: litellm-rust
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- run: rustup toolchain install --no-self-update
- uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8
with:
tool: cargo-nextest@0.9.143
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
workspaces: litellm-rust
cache-on-failure: true
- run: cargo nextest run --workspace --locked
- run: cargo test --workspace --doc --locked
rust-wheel:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
@ -114,18 +138,10 @@ jobs:
- run: rustup toolchain install --no-self-update
- uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
path: |
~/.cargo/registry
~/.cargo/git
litellm-rust/target
key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-${{ github.job }}-
- run: cargo test --workspace --locked
working-directory: litellm-rust
workspaces: litellm-rust
cache-on-failure: true
- run: uv build --wheel --out-dir dist

52
litellm-rust/crates/python-bridge/src/lifecycle/mod.rs
View file

@ -599,6 +599,34 @@ mod tests {
static PYTHON_GLOBALS: Mutex<()> = Mutex::new(());
fn install_lifecycle_module(py: Python<'_>) -> Bound<'_, PyModule> {
py.run(
pyo3::ffi::c_str!(
r#"
import sys
import types
sys.modules.setdefault('litellm', types.ModuleType('litellm'))
sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bridge'))
"#
),
None,
None,
)
.unwrap();
let source = std::ffi::CString::new(include_str!(
"../../../../../litellm/rust_bridge/lifecycle.py"
))
.unwrap();
PyModule::from_code(
py,
&source,
pyo3::ffi::c_str!("lifecycle.py"),
pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"),
)
.unwrap()
}
fn install_logging_worker(py: Python<'_>, worker: &Bound<'_, PyAny>) -> PyResult<()> {
py.import("litellm.litellm_core_utils.logging_worker")?
.setattr("GLOBAL_LOGGING_WORKER", worker)
@ -773,17 +801,7 @@ mod tests {
.unwrap_or_else(|error| error.into_inner());
Python::initialize();
Python::attach(|py| {
let source = std::ffi::CString::new(include_str!(
"../../../../../litellm/rust_bridge/lifecycle.py"
))
.unwrap();
PyModule::from_code(
py,
&source,
pyo3::ffi::c_str!("lifecycle.py"),
pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"),
)
.unwrap();
install_lifecycle_module(py);
let route = SyntheticRoute(
PythonCallState::new(
py,
@ -819,17 +837,7 @@ mod tests {
Python::initialize();
Python::attach(|py| {
py.import("asyncio").unwrap();
let source = std::ffi::CString::new(include_str!(
"../../../../../litellm/rust_bridge/lifecycle.py"
))
.unwrap();
let module = PyModule::from_code(
py,
&source,
pyo3::ffi::c_str!("lifecycle.py"),
pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"),
)
.unwrap();
let module = install_lifecycle_module(py);
let locals = PyDict::new(py);
locals
.set_item("drive", module.getattr("drive").unwrap())

1
litellm-rust/crates/python-bridge/src/marshal.rs
View file

@ -190,6 +190,7 @@ mod tests {
#[test]
fn required_shapes_preserve_nested_values_and_existing_errors() {
Python::initialize();
let nested = json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]);
assert_eq!(
Value::Array(required_array("messages", nested.clone()).unwrap()),

19
litellm-rust/crates/token-counter/src/tiktoken.rs
View file

@ -195,14 +195,21 @@ mod tests {
}
#[test]
fn long_repeated_runs_stay_cheap() {
fn long_repeated_runs_cost_close_to_linear() {
let ranks = ranks();
let mut scratch = MergeScratch::default();
let piece = vec![b' '; 1 << 20];
let started = std::time::Instant::now();
let count = ranks.count_piece(&piece, &mut scratch);
assert!(count > 0);
assert!(started.elapsed().as_secs() < 5, "{:?}", started.elapsed());
let mut time = |len: usize| {
let piece = vec![b' '; len];
let started = std::time::Instant::now();
assert!(ranks.count_piece(&piece, &mut scratch) > 0);
started.elapsed()
};
let small = (0..3).map(|_| time(1 << 14)).min().unwrap();
let large = time(1 << 18);
assert!(
large < small * 64,
"{small:?} for 2^14 bytes, {large:?} for 2^18"
);
}
#[test]

15
litellm/integrations/opentelemetry.py
View file

@ -5,6 +5,7 @@ from collections.abc import Callable, Iterable, Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
import litellm
@ -21,7 +22,9 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
parse_semconv_opt_in,
)
from litellm.integrations.otel.mappers.utils import drop_none
from litellm.integrations.otel.model.baggage import promoted_metadata
from litellm.integrations.otel.model.db_endpoint import db_span_attributes
from litellm.integrations.otel.model.metadata import flatten_metadata
from litellm.integrations.otel.model.semconv import Metric
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -303,6 +306,7 @@ class OpenTelemetryConfig:
# under ``litellm.team.metadata``. Empty by default so none of a team's
# metadata leaves the process until explicitly allowlisted.
baggage_team_metadata_keys: list[str] = field(default_factory=list)
baggage_metadata_keys: list[str] = field(default_factory=list)
# Prometheus-style include/exclude control over which attributes are stamped
# on emitted metrics, to cap metric cardinality.
attributes: OTELMetricAttributeFilter | None = None
@ -329,6 +333,9 @@ class OpenTelemetryConfig:
self.baggage_team_metadata_keys = _normalize_team_metadata_keys(
self.baggage_team_metadata_keys
) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS"))
self.baggage_metadata_keys = _normalize_team_metadata_keys(
self.baggage_metadata_keys
) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_METADATA_KEYS"))
@classmethod
def from_env(cls):
@ -381,11 +388,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
**kwargs,
):
team_metadata_keys_override: Final = kwargs.pop("baggage_team_metadata_keys", None)
metadata_keys_override: Final = kwargs.pop("baggage_metadata_keys", None)
metric_attributes_override: Final = kwargs.pop("attributes", None)
if config is None:
config = OpenTelemetryConfig.from_env()
if team_metadata_keys_override is not None:
config.baggage_team_metadata_keys = _normalize_team_metadata_keys(team_metadata_keys_override)
if metadata_keys_override is not None:
config.baggage_metadata_keys = _normalize_team_metadata_keys(metadata_keys_override)
if metric_attributes_override is not None:
config.attributes = _build_metric_attribute_filter(metric_attributes_override)
@ -1557,6 +1567,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if team_metadata:
self.safe_set_attribute(span=span, key=TEAM_METADATA_ATTRIBUTE, value=team_metadata)
if self.config.baggage_metadata_keys:
flat_metadata: Final = MappingProxyType(dict(flatten_metadata(metadata)))
for key, value in promoted_metadata(flat_metadata, tuple(self.config.baggage_metadata_keys)).items():
self.safe_set_attribute(span=span, key=key, value=value)
model_group: Final = standard_logging_payload.get("model_group")
if model_group:
self.safe_set_attribute(span=span, key=MODEL_GROUP_ATTRIBUTE, value=model_group)

11
litellm/integrations/otel/logger.py
View file

@ -33,6 +33,7 @@ from litellm.integrations.otel.model.metadata import (
LLMCallEvent,
RequestIdentity,
auth_metadata,
metadata_from_request_data,
model_from_request_data,
)
from litellm.integrations.otel.model.payloads import (
@ -679,7 +680,12 @@ class OpenTelemetryV2(CustomLogger):
# / errors are the FastAPI instrumentor's job, so we don't touch it here.
# ====================================================================== #
def seed_request_identity(self, user_api_key_dict: object, model: str | None = None) -> None:
def seed_request_identity(
self,
user_api_key_dict: object,
model: str | None = None,
request_metadata: Mapping[str, object] | None = None,
) -> None:
"""Attach request-identity Baggage to the current context + server span.
Seeding identity into Baggage makes **every** span emitted afterwards for
@ -691,7 +697,7 @@ class OpenTelemetryV2(CustomLogger):
isn't determined yet, which is correct.
"""
try:
identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict)
identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict, request_metadata)
bag: Final = promoted_baggage(
identity,
model,
@ -743,6 +749,7 @@ class OpenTelemetryV2(CustomLogger):
self.seed_request_identity(
user_api_key_dict,
model=model_from_request_data(data),
request_metadata=metadata_from_request_data(data),
)
return data

31
litellm/integrations/otel/model/baggage.py
View file

@ -15,9 +15,10 @@ never promoted whole.
import json
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import Final
from litellm.integrations.otel.model.metadata import RequestIdentity
from litellm.integrations.otel.model.metadata import REQUESTER_METADATA_PATH, RequestIdentity
from litellm.integrations.otel.model.semconv import GenAI, LiteLLM
# Attribute key -> value extractor over (identity, request_model,
@ -79,17 +80,23 @@ def promoted_baggage(
``team_metadata_keys`` selects sub-keys of the team's metadata to promote
under ``litellm.team.metadata``. Empty values are dropped.
"""
out: Final[dict[str, str]] = {}
for key, extract in _PROMOTABLE.items():
if key in promoted_keys:
value = extract(identity, request_model, team_metadata_keys)
if value:
out[key] = value
for meta_key in metadata_keys:
value = identity.metadata.get(meta_key)
if value:
out[f"{LiteLLM.METADATA_PREFIX}{meta_key}"] = value
return out
identity_values: Final = {
key: value
for key, extract in _PROMOTABLE.items()
if key in promoted_keys and (value := extract(identity, request_model, team_metadata_keys))
}
return {**identity_values, **promoted_metadata(identity.metadata, metadata_keys)}
def promoted_metadata(metadata: Mapping[str, str], metadata_keys: tuple[str, ...]) -> Mapping[str, str]:
"""Allowlisted entries of a flattened metadata mapping under ``litellm.metadata.*``."""
return MappingProxyType(
{
f"{LiteLLM.METADATA_PREFIX}{meta_key.removeprefix(REQUESTER_METADATA_PATH)}": value
for meta_key in metadata_keys
if (value := metadata.get(meta_key))
}
)
def _filtered_team_metadata_json(

5
litellm/integrations/otel/model/config.py
View file

@ -210,7 +210,10 @@ class OpenTelemetryV2Config(BaseSettings):
validation_alias=AliasChoices("baggage_metadata_keys", "LITELLM_OTEL_BAGGAGE_METADATA_KEYS"),
description=(
"Metadata sub-keys promoted under the ``litellm.metadata.*`` "
"namespace. Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` "
"namespace. A dotted path such as ``requester_metadata.trace_id`` "
"reads the caller's nested ``metadata.trace_id`` and is promoted as "
"``litellm.metadata.trace_id``; other dotted keys keep their full path. "
"Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` "
"env var (comma-separated) or "
"``callback_settings.otel.baggage_metadata_keys`` in config.yaml."
),

49
litellm/integrations/otel/model/metadata.py
View file

@ -49,6 +49,8 @@ if TYPE_CHECKING:
from litellm.types.utils import StandardLoggingPayload
LANGFUSE_TRACE_NAME_HEADER: Final = "langfuse_trace_name"
REQUESTER_METADATA_KEY: Final = "requester_metadata"
REQUESTER_METADATA_PATH: Final = f"{REQUESTER_METADATA_KEY}."
@dataclass(frozen=True)
@ -78,7 +80,7 @@ class RequestIdentity:
model, not just the user-facing one.
"""
raw_meta: Final = cast(Mapping[str, object], payload.get("metadata") or {})
metadata = {key: str(value) for key, value in raw_meta.items() if isinstance(value, (str, bool, int, float))}
metadata: Final = MappingProxyType(dict(flatten_metadata(raw_meta)))
return cls(
call_id=as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")),
# StandardLoggingMetadata's canonical key is ``user_api_key_team_id``;
@ -95,7 +97,9 @@ class RequestIdentity:
)
@classmethod
def from_user_api_key_auth(cls, auth: object) -> RequestIdentity:
def from_user_api_key_auth(
cls, auth: object, request_metadata: Mapping[str, object] | None = None
) -> RequestIdentity:
"""Identity from a ``UserAPIKeyAuth`` (duck-typed to keep this module
free of a proxy import).
@ -103,11 +107,13 @@ class RequestIdentity:
guardrail, or service span is created — so the whole request's spans
inherit identity, not just the LLM-call span. Metadata sub-keys use the
``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS``
promotes.
promotes; ``request_metadata`` (the caller's ``requester_metadata``
snapshot) is flattened to dotted keys so ``requester_metadata.<key>``
resolves too.
"""
get: Final = lambda name: getattr(auth, name, None) # noqa: E731
metadata: Final = {
meta_key: str(value)
auth_meta: Final = tuple(
(meta_key, str(value))
for meta_key, attr in (
("user_api_key_user_id", "user_id"),
("user_api_key_org_id", "org_id"),
@ -115,7 +121,9 @@ class RequestIdentity:
("user_api_key_end_user_id", "end_user_id"),
)
if (value := get(attr))
}
)
request_meta: Final = flatten_metadata(request_metadata) if request_metadata is not None else ()
metadata: Final = MappingProxyType(dict((*request_meta, *auth_meta)))
return cls(
team_id=as_str(get("team_id")),
team_alias=as_str(get("team_alias")),
@ -351,6 +359,35 @@ def model_from_request_data(data: object) -> str | None:
return None
def metadata_from_request_data(data: object) -> Mapping[str, object] | None:
"""The caller's ``requester_metadata`` snapshot from a pre-call ``data`` dict, keyed under its wrapper.
The proxy stores it under ``metadata`` or ``litellm_metadata`` depending on the route;
the proxy-owned siblings (``user_api_key_*``, ``requester_ip_address``) are not read.
"""
top: Final = _as_str_mapping(data)
if top is None:
return None
snapshots: Final = tuple(
snapshot
for name in ("metadata", "litellm_metadata")
if (nested := _as_str_mapping(top.get(name))) is not None
and (snapshot := _as_str_mapping(nested.get(REQUESTER_METADATA_KEY))) is not None
)
return MappingProxyType({REQUESTER_METADATA_KEY: snapshots[0]}) if snapshots else None
def flatten_metadata(raw: Mapping[str, object]) -> Iterator[tuple[str, str]]:
"""Scalar leaves of a nested metadata mapping, keyed by their dotted path."""
stack: Final = list(tuple(raw.items())[::-1]) # mutable-ok: iterative worklist keeps the walk off the call stack
while stack:
key, value = stack.pop()
if (nested := _as_str_mapping(value)) is not None:
stack.extend(tuple((f"{key}.{sub_key}", sub_value) for sub_key, sub_value in nested.items())[::-1])
elif isinstance(value, (str, bool, int, float)):
yield key, str(value)
def resolve_provider_model(payload: StandardLoggingPayload) -> str | None:
"""The model litellm dispatched to the provider, from the payload.

21
litellm/llms/bedrock/vector_stores/transformation.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from copy import deepcopy
from typing import TYPE_CHECKING, Any, Final, cast
from urllib.parse import urlparse
@ -14,6 +15,7 @@ from litellm.types.integrations.rag.bedrock_knowledgebase import (
BedrockKBResponse,
BedrockKBRetrievalConfiguration,
BedrockKBRetrievalQuery,
BedrockKBUserContext,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.vector_stores import (
@ -242,10 +244,29 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
retrieval_config.setdefault("vectorSearchConfiguration", {})["filter"] = filters
if retrieval_config:
request_body["retrievalConfiguration"] = cast(BedrockKBRetrievalConfiguration, retrieval_config)
user_context: Final = self._user_context(extra_body=extra_body, litellm_params=litellm_params)
if user_context is not None:
request_body["userContext"] = user_context
litellm_logging_obj.model_call_details["query"] = query
return url, request_body
@staticmethod
def _user_context(
extra_body: Mapping[str, object] | None, litellm_params: Mapping[str, object]
) -> BedrockKBUserContext | None:
sources: Final = tuple(source for source in (extra_body, litellm_params) if isinstance(source, Mapping))
found: Final = next(
(
source[key]
for source in sources
for key in ("userContext", "user_context")
if source.get(key) is not None
),
None,
)
return None if found is None else cast(BedrockKBUserContext, found)
def sign_request(
self,
headers: dict,

20
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -49,6 +49,17 @@ if TYPE_CHECKING:
import tiktoken
def _map_reasoning_effort(value: object) -> object:
effort: Final[object] = cast(Mapping[str, object], value).get("effort") if isinstance(value, Mapping) else value
if effort is True:
return "medium"
if effort is False:
return "none"
if effort == "auto":
return None
return effort
def _extract_fireworks_hidden_params(payload: dict) -> dict:
"""
Collect Fireworks-specific response fields (perf_metrics, prompt_token_ids,
@ -327,12 +338,9 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
elif param == "max_completion_tokens":
optional_params["max_tokens"] = value
elif param == "reasoning_effort":
if value is True:
optional_params["reasoning_effort"] = "medium"
elif value is False:
optional_params["reasoning_effort"] = "none"
elif value != "auto":
optional_params["reasoning_effort"] = value
effort = _map_reasoning_effort(value)
if effort is not None:
optional_params["reasoning_effort"] = effort
elif param in supported_openai_params:
if value is not None:
optional_params[param] = value

124
litellm/model_prices_and_context_window_backup.json
View file

@ -11313,13 +11313,16 @@
},
"azure_ai/grok-4.3": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 200000,
"max_output_tokens": 200000,
"max_tokens": 200000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -11331,13 +11334,16 @@
},
"azure_ai/grok-4.6": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_200k_tokens": 4e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"output_cost_per_token_above_200k_tokens": 1.2e-05,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -26162,7 +26168,9 @@
"output_vector_size": 3072,
"rpm": 10000,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supports_audio_input": true,
"supports_multimodal": true,
"supports_vision": true,
"tpm": 10000000
},
"gemini/gemini-1.5-flash": {
@ -26303,8 +26311,8 @@
"input_cost_per_token": 3e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 2.5e-06,
"output_cost_per_token": 2.5e-06,
@ -26350,6 +26358,7 @@
"output_cost_per_token_batches": 1.25e-06,
"output_cost_per_token_flex": 1.25e-06,
"output_cost_per_token_priority": 4.5e-06,
"supports_audio_input": true,
"supports_image_size": false
},
"gemini/gemini-2.5-flash-image": {
@ -26362,7 +26371,7 @@
"input_cost_per_token_priority": 5.4e-07,
"litellm_provider": "gemini",
"supports_reasoning": false,
"max_input_tokens": 32768,
"max_input_tokens": 65536,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
@ -26388,22 +26397,23 @@
"image"
],
"supports_audio_output": false,
"supports_function_calling": true,
"supports_function_calling": false,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"supports_web_search": false,
"tpm": 8000000,
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"supports_audio_input": false,
"supports_image_size": false
},
"gemini/gemini-3-pro-image": {
@ -26441,7 +26451,7 @@
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
@ -26544,7 +26554,7 @@
"input_cost_per_token": 5e-07,
"input_cost_per_token_batches": 2.5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 65536,
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
@ -26571,7 +26581,7 @@
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_reasoning": false,
"supports_response_schema": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
@ -26653,12 +26663,13 @@
"text",
"image"
],
"supports_function_calling": true,
"supports_function_calling": false,
"supports_prompt_caching": false,
"supports_reasoning": false,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": false,
"tpm": 4000000
},
"gemini/deep-research-pro-preview-12-2025": {
@ -26711,8 +26722,8 @@
"input_cost_per_token": 1e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 4e-07,
"output_cost_per_token": 4e-07,
@ -26758,6 +26769,7 @@
"output_cost_per_token_batches": 2e-07,
"output_cost_per_token_flex": 2e-07,
"output_cost_per_token_priority": 7.2e-07,
"supports_audio_input": true,
"supports_image_size": false
},
"gemini/gemini-2.5-flash-lite-preview-09-2025": {
@ -27005,6 +27017,9 @@
"input_cost_per_token": 5e-07,
"input_cost_per_token_batches": 2.5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "audio_speech",
"output_cost_per_audio_token": 1e-05,
"output_cost_per_token": 1e-05,
@ -27013,7 +27028,11 @@
"/v1/audio/speech"
],
"tpm": 4000000,
"rpm": 10
"rpm": 10,
"supports_audio_input": false,
"supports_function_calling": false,
"supports_response_schema": false,
"supports_web_search": false
},
"gemini/gemini-2.5-pro": {
"cache_read_input_token_cost": 1.25e-07,
@ -27027,8 +27046,8 @@
"input_cost_per_token_above_200k_tokens_priority": 4.5e-06,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1e-05,
"output_cost_per_token_above_200k_tokens": 1.5e-05,
@ -27338,8 +27357,8 @@
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 3e-06,
"output_cost_per_token": 3e-06,
@ -27388,7 +27407,8 @@
"input_cost_per_token_batches": 2.5e-07,
"input_cost_per_token_flex": 2.5e-07,
"output_cost_per_token_batches": 1.5e-06,
"output_cost_per_token_flex": 1.5e-06
"output_cost_per_token_flex": 1.5e-06,
"supports_audio_input": true
},
"gemini/gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
@ -27397,8 +27417,8 @@
"input_cost_per_token": 1.5e-06,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
@ -28112,9 +28132,9 @@
"input_cost_per_token": 1e-06,
"input_cost_per_token_batches": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_input_tokens": 8192,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2e-05,
@ -28127,19 +28147,20 @@
"audio"
],
"supports_audio_output": false,
"supports_function_calling": true,
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_vision": false,
"supports_web_search": false,
"tpm": 10000000,
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
}
},
"supports_audio_input": false
},
"gemini/gemini-exp-1114": {
"input_cost_per_token": 0,
@ -55917,7 +55938,7 @@
"input_cost_per_audio_token": 3e-06,
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_input_tokens": 131072,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "realtime",
@ -55937,7 +55958,11 @@
],
"supports_audio_input": true,
"supports_audio_output": true,
"gemini_native_audio": true
"gemini_native_audio": true,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_vision": true,
"supports_web_search": true
},
"gemini-3.1-flash-live-preview": {
"input_cost_per_audio_token": 3e-06,
@ -55971,7 +55996,8 @@
"supports_vision": true,
"supports_web_search": true,
"gemini_audio_only_live": true,
"input_cost_per_second": 8.33333333333e-05
"input_cost_per_second": 8.33333333333e-05,
"supports_response_schema": false
},
"gemini/gemini-2.5-flash-native-audio-latest": {
"input_cost_per_audio_token": 3e-06,
@ -56033,7 +56059,7 @@
"input_cost_per_audio_token": 3e-06,
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_input_tokens": 131072,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "realtime",
@ -56055,7 +56081,11 @@
"supports_audio_output": true,
"tpm": 250000,
"rpm": 10,
"gemini_native_audio": true
"gemini_native_audio": true,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_vision": true,
"supports_web_search": true
},
"gemini/gemini-3.1-flash-live-preview": {
"input_cost_per_audio_token": 3e-06,
@ -56091,7 +56121,8 @@
"tpm": 250000,
"rpm": 10,
"gemini_audio_only_live": true,
"input_cost_per_second": 8.33333333333e-05
"input_cost_per_second": 8.33333333333e-05,
"supports_response_schema": false
},
"gemini/gemini-3.1-flash-tts-preview": {
"input_cost_per_token": 1e-06,
@ -56108,19 +56139,29 @@
"/v1/audio/speech"
],
"tpm": 4000000,
"rpm": 10
"rpm": 10,
"supports_function_calling": false,
"supports_response_schema": false,
"supports_web_search": false
},
"gemini-2.5-flash-preview-tts": {
"input_cost_per_token": 5e-07,
"input_cost_per_token_batches": 2.5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "audio_speech",
"output_cost_per_audio_token": 1e-05,
"output_cost_per_token": 1e-05,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
],
"supports_audio_input": false,
"supports_function_calling": false,
"supports_response_schema": false,
"supports_web_search": false
},
"gemini-flash-latest": {
"cache_read_input_token_cost": 3e-08,
@ -58694,6 +58735,9 @@
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": false,
"supports_response_schema": false,
"supports_web_search": false,
"tpm": 250000
},
"gemini/gemini-3.5-transcribe": {
@ -58715,7 +58759,8 @@
],
"supports_audio_input": true,
"tpm": 800000,
"rpm": 2000
"rpm": 2000,
"supports_function_calling": false
},
"gemini/gemini-3.5-transcribe-live": {
"input_cost_per_audio_token": 3.5e-06,
@ -58735,7 +58780,8 @@
],
"supports_audio_input": true,
"tpm": 250000,
"rpm": 10
"rpm": 10,
"supports_function_calling": false
},
"vertex_ai/gemini-3.5-transcribe-preview": {
"input_cost_per_audio_token": 2e-06,
@ -61374,7 +61420,7 @@
"input_cost_per_audio_token": 1.5e-06,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "gemini",
"max_input_tokens": 131072,
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",

13
litellm/proxy/auth/user_api_key_auth.py
View file

@ -2571,6 +2571,13 @@ def _token_can_vouch_for_team(valid_token: UserAPIKeyAuth, lookup_error: BaseExc
return PrismaDBExceptionHandler.should_allow_request_on_db_unavailable()
def is_no_auth_dev_mode(master_key: str | None, general_settings: Mapping[str, object]) -> bool:
return master_key is None and not any(
general_settings.get(flag, False)
for flag in ("enable_jwt_auth", "enable_oauth2_auth", "enable_oauth2_proxy_auth")
)
@tracer.wrap()
async def _run_centralized_common_checks(
user_api_key_auth_obj: UserAPIKeyAuth,
@ -2630,11 +2637,7 @@ async def _run_centralized_common_checks(
# Running common_checks would block every admin route on these
# deployments where that was previously not the contract. If any
# authn is enabled (JWT, OAuth2, OAuth2-proxy), authz must run.
if master_key is None and not (
general_settings.get("enable_jwt_auth", False)
or general_settings.get("enable_oauth2_auth", False)
or general_settings.get("enable_oauth2_proxy_auth", False)
):
if is_no_auth_dev_mode(master_key, general_settings):
return
if user_custom_auth is not None and not general_settings.get("custom_auth_run_common_checks", False):

69
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -45,6 +45,7 @@ from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.auth.user_api_key_auth import (
_get_bearer_token,
is_no_auth_dev_mode,
user_api_key_auth,
user_api_key_auth_websocket,
)
@ -709,8 +710,7 @@ async def anthropic_proxy_route(
endpoint_func: Final = create_pass_through_route(
endpoint=endpoint,
target=str(updated_url),
custom_headers=auth_header if auth_header is not None else {},
_forward_headers=True,
custom_headers=_upstream_headers_for_anthropic_route(request, user_api_key_dict, auth_header),
is_streaming_request=is_streaming_request,
) # dynamically construct pass-through endpoint based on incoming path
received_value: Final = await endpoint_func(
@ -1989,6 +1989,19 @@ _HEADERS_NEVER_FORWARDED_TO_VERTEX: Final = frozenset({"content-length", "host"}
SpecialHeaders.litellm_credential_header_names() - _VERTEX_UPSTREAM_CREDENTIAL_HEADERS
)
_CREDENTIALLESS_ANTHROPIC_MISSING_CREDENTIAL_DETAIL: Final = (
"No Anthropic credential is configured on this proxy and the request carried no upstream "
"Anthropic credential. The LiteLLM virtual key is not forwarded to Anthropic. Configure an "
"Anthropic credential (ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN, or a model with "
"use_in_pass_through: true), or send your own Anthropic API key in the x-api-key header or "
"your own Anthropic OAuth token in the Authorization header."
)
_ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS: Final = frozenset({"authorization", "x-api-key"})
_HEADERS_NEVER_FORWARDED_TO_ANTHROPIC: Final = frozenset({"content-length", "host", "accept-encoding"}) | (
SpecialHeaders.litellm_credential_header_names() - _ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS
)
_MAPPED_ROUTE_CALLER_KEY_HEADER: Final = "litellm_user_api_key"
@ -2026,8 +2039,11 @@ def _is_authenticated_caller_jwt(value: str, jwt_claims: Mapping[str, object]) -
def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAuth) -> bool:
"""Whether a header value is the master key, the JWT that authenticated, or the key stored as ``api_key``."""
from litellm.proxy.proxy_server import master_key
"""Whether a header value is the master key, the JWT that authenticated, or the key stored as ``api_key``.
A proxy in no-auth dev mode without custom auth authenticated nothing, so none of the caller's values is one.
"""
from litellm.proxy.proxy_server import general_settings, master_key, user_custom_auth
normalized: Final = _normalize_credential_value(value)
if master_key is not None and hmac.compare_digest(normalized.encode(), master_key.encode()):
@ -2035,35 +2051,54 @@ def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAut
jwt_claims: Final = user_api_key_dict.jwt_claims
if jwt_claims and _is_authenticated_caller_jwt(normalized, jwt_claims):
return True
if is_no_auth_dev_mode(master_key, general_settings) and user_custom_auth is None:
return False
authenticated_key: Final = user_api_key_dict.api_key
if authenticated_key is None:
return False
if master_key is None and not normalized.startswith("sk-"):
return False
stored_representation: Final = UserAPIKeyAuth._safe_hash_litellm_api_key(normalized) # pyright: ignore[reportPrivateUsage] # the exact transform auth applied when it stored api_key
return hmac.compare_digest(stored_representation.encode(), authenticated_key.encode())
def _caller_headers_without_litellm_secrets(
request: Request, user_api_key_dict: UserAPIKeyAuth, never_forwarded: frozenset[str]
) -> Mapping[str, str]:
incoming: Final = _safe_get_request_headers(request)
dropped_by_name: Final = never_forwarded.union(
(_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names())
)
return MappingProxyType(
{
name: value
for name, value in incoming.items()
if name not in dropped_by_name and not _is_authenticated_caller_secret(value, user_api_key_dict)
}
)
def _forwarded_headers_for_credentialless_vertex_passthrough(
request: Request, user_api_key_dict: UserAPIKeyAuth
) -> Mapping[str, str]:
"""Caller headers to forward on the bring-your-own-credentials Vertex branch, minus LiteLLM secrets."""
incoming: Final = _safe_get_request_headers(request)
never_forwarded: Final = _HEADERS_NEVER_FORWARDED_TO_VERTEX.union(
(_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names())
forwarded: Final = _caller_headers_without_litellm_secrets(
request, user_api_key_dict, _HEADERS_NEVER_FORWARDED_TO_VERTEX
)
forwarded: Final = MappingProxyType(
{
name: value
for name, value in incoming.items()
if name not in never_forwarded and not _is_authenticated_caller_secret(value, user_api_key_dict)
}
)
if "authorization" not in forwarded and "x-goog-api-key" not in forwarded:
if _VERTEX_UPSTREAM_CREDENTIAL_HEADERS.isdisjoint(forwarded):
raise HTTPException(status_code=401, detail=_CREDENTIALLESS_VERTEX_MISSING_CREDENTIAL_DETAIL)
return forwarded
def _upstream_headers_for_anthropic_route(
request: Request, user_api_key_dict: UserAPIKeyAuth, proxy_auth_header: Mapping[str, str] | None
) -> Mapping[str, str]:
caller_headers: Final = _caller_headers_without_litellm_secrets(
request, user_api_key_dict, _HEADERS_NEVER_FORWARDED_TO_ANTHROPIC
)
if proxy_auth_header is None and _ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS.isdisjoint(caller_headers):
raise HTTPException(status_code=401, detail=_CREDENTIALLESS_ANTHROPIC_MISSING_CREDENTIAL_DETAIL)
return MappingProxyType({**caller_headers, **(proxy_auth_header or {})})
async def _prepare_vertex_auth_headers(
request: Request,
vertex_credentials: VertexPassThroughCredentials | None,

6
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -176,6 +176,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
return {**item_kwargs, "name": tool_name, **namespace_kwargs}
def _is_reasoning_end(self, chunk):
if not chunk.choices:
return False
delta: Final = chunk.choices[0].delta
# if this indicates reasoning content, don't consider reasoning ended
@ -897,6 +899,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
# Change: Never return a value, just enqueue output item events
if self.sent_output_item_added_event:
return
if not chunk.choices:
return
delta: Final = chunk.choices[0].delta
self._sequence_number += 1
@ -1224,6 +1228,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
It's unclear how users expect litellm to translate multiple-choices-per-chunk to the responses API output.
"""
if not choices:
return ""
choice: Final = choices[0]
chat_completion_delta: Final[ChatCompletionDelta] = choice.delta
return chat_completion_delta.content or ""

7
litellm/types/integrations/rag/bedrock_knowledgebase.py
View file

@ -1,6 +1,6 @@
from typing import Any, Literal
from typing_extensions import TypedDict
from typing_extensions import ReadOnly, TypedDict
class BedrockKBLocation(TypedDict, total=False):
@ -127,6 +127,10 @@ class BedrockKBGuardrailConfiguration(TypedDict, total=False):
guardrailVersion: str | None
class BedrockKBUserContext(TypedDict):
userId: ReadOnly[str]
class BedrockKBRequest(TypedDict, total=False):
"""Complete request structure for Bedrock Knowledge Base retrieval."""
@ -134,6 +138,7 @@ class BedrockKBRequest(TypedDict, total=False):
nextToken: str | None
retrievalConfiguration: BedrockKBRetrievalConfiguration | None
retrievalQuery: BedrockKBRetrievalQuery
userContext: ReadOnly[BedrockKBUserContext | None]
#########################################################################

124
model_prices_and_context_window.json
View file

@ -11313,13 +11313,16 @@
},
"azure_ai/grok-4.3": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 200000,
"max_output_tokens": 200000,
"max_tokens": 200000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -11331,13 +11334,16 @@
},
"azure_ai/grok-4.6": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_200k_tokens": 4e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"output_cost_per_token_above_200k_tokens": 1.2e-05,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -26162,7 +26168,9 @@
"output_vector_size": 3072,
"rpm": 10000,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supports_audio_input": true,
"supports_multimodal": true,
"supports_vision": true,
"tpm": 10000000
},
"gemini/gemini-1.5-flash": {
@ -26303,8 +26311,8 @@
"input_cost_per_token": 3e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 2.5e-06,
"output_cost_per_token": 2.5e-06,
@ -26350,6 +26358,7 @@
"output_cost_per_token_batches": 1.25e-06,
"output_cost_per_token_flex": 1.25e-06,
"output_cost_per_token_priority": 4.5e-06,
"supports_audio_input": true,
"supports_image_size": false
},
"gemini/gemini-2.5-flash-image": {
@ -26362,7 +26371,7 @@
"input_cost_per_token_priority": 5.4e-07,
"litellm_provider": "gemini",
"supports_reasoning": false,
"max_input_tokens": 32768,
"max_input_tokens": 65536,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
@ -26388,22 +26397,23 @@
"image"
],
"supports_audio_output": false,
"supports_function_calling": true,
"supports_function_calling": false,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"supports_web_search": false,
"tpm": 8000000,
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"supports_audio_input": false,
"supports_image_size": false
},
"gemini/gemini-3-pro-image": {
@ -26441,7 +26451,7 @@
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
@ -26544,7 +26554,7 @@
"input_cost_per_token": 5e-07,
"input_cost_per_token_batches": 2.5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 65536,
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
@ -26571,7 +26581,7 @@
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_reasoning": false,
"supports_response_schema": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
@ -26653,12 +26663,13 @@
"text",
"image"
],
"supports_function_calling": true,
"supports_function_calling": false,
"supports_prompt_caching": false,
"supports_reasoning": false,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": false,
"tpm": 4000000
},
"gemini/deep-research-pro-preview-12-2025": {
@ -26711,8 +26722,8 @@
"input_cost_per_token": 1e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 4e-07,
"output_cost_per_token": 4e-07,
@ -26758,6 +26769,7 @@
"output_cost_per_token_batches": 2e-07,
"output_cost_per_token_flex": 2e-07,
"output_cost_per_token_priority": 7.2e-07,
"supports_audio_input": true,
"supports_image_size": false
},
"gemini/gemini-2.5-flash-lite-preview-09-2025": {
@ -27005,6 +27017,9 @@
"input_cost_per_token": 5e-07,
"input_cost_per_token_batches": 2.5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "audio_speech",
"output_cost_per_audio_token": 1e-05,
"output_cost_per_token": 1e-05,
@ -27013,7 +27028,11 @@
"/v1/audio/speech"
],
"tpm": 4000000,
"rpm": 10
"rpm": 10,
"supports_audio_input": false,
"supports_function_calling": false,
"supports_response_schema": false,
"supports_web_search": false
},
"gemini/gemini-2.5-pro": {
"cache_read_input_token_cost": 1.25e-07,
@ -27027,8 +27046,8 @@
"input_cost_per_token_above_200k_tokens_priority": 4.5e-06,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1e-05,
"output_cost_per_token_above_200k_tokens": 1.5e-05,
@ -27338,8 +27357,8 @@
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 3e-06,
"output_cost_per_token": 3e-06,
@ -27388,7 +27407,8 @@
"input_cost_per_token_batches": 2.5e-07,
"input_cost_per_token_flex": 2.5e-07,
"output_cost_per_token_batches": 1.5e-06,
"output_cost_per_token_flex": 1.5e-06
"output_cost_per_token_flex": 1.5e-06,
"supports_audio_input": true
},
"gemini/gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
@ -27397,8 +27417,8 @@
"input_cost_per_token": 1.5e-06,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
@ -28112,9 +28132,9 @@
"input_cost_per_token": 1e-06,
"input_cost_per_token_batches": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_input_tokens": 8192,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2e-05,
@ -28127,19 +28147,20 @@
"audio"
],
"supports_audio_output": false,
"supports_function_calling": true,
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_vision": false,
"supports_web_search": false,
"tpm": 10000000,
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
}
},
"supports_audio_input": false
},
"gemini/gemini-exp-1114": {
"input_cost_per_token": 0,
@ -55917,7 +55938,7 @@
"input_cost_per_audio_token": 3e-06,
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_input_tokens": 131072,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "realtime",
@ -55937,7 +55958,11 @@
],
"supports_audio_input": true,
"supports_audio_output": true,
"gemini_native_audio": true
"gemini_native_audio": true,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_vision": true,
"supports_web_search": true
},
"gemini-3.1-flash-live-preview": {
"input_cost_per_audio_token": 3e-06,
@ -55971,7 +55996,8 @@
"supports_vision": true,
"supports_web_search": true,
"gemini_audio_only_live": true,
"input_cost_per_second": 8.33333333333e-05
"input_cost_per_second": 8.33333333333e-05,
"supports_response_schema": false
},
"gemini/gemini-2.5-flash-native-audio-latest": {
"input_cost_per_audio_token": 3e-06,
@ -56033,7 +56059,7 @@
"input_cost_per_audio_token": 3e-06,
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_input_tokens": 131072,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "realtime",
@ -56055,7 +56081,11 @@
"supports_audio_output": true,
"tpm": 250000,
"rpm": 10,
"gemini_native_audio": true
"gemini_native_audio": true,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_vision": true,
"supports_web_search": true
},
"gemini/gemini-3.1-flash-live-preview": {
"input_cost_per_audio_token": 3e-06,
@ -56091,7 +56121,8 @@
"tpm": 250000,
"rpm": 10,
"gemini_audio_only_live": true,
"input_cost_per_second": 8.33333333333e-05
"input_cost_per_second": 8.33333333333e-05,
"supports_response_schema": false
},
"gemini/gemini-3.1-flash-tts-preview": {
"input_cost_per_token": 1e-06,
@ -56108,19 +56139,29 @@
"/v1/audio/speech"
],
"tpm": 4000000,
"rpm": 10
"rpm": 10,
"supports_function_calling": false,
"supports_response_schema": false,
"supports_web_search": false
},
"gemini-2.5-flash-preview-tts": {
"input_cost_per_token": 5e-07,
"input_cost_per_token_batches": 2.5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "audio_speech",
"output_cost_per_audio_token": 1e-05,
"output_cost_per_token": 1e-05,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
],
"supports_audio_input": false,
"supports_function_calling": false,
"supports_response_schema": false,
"supports_web_search": false
},
"gemini-flash-latest": {
"cache_read_input_token_cost": 3e-08,
@ -58694,6 +58735,9 @@
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": false,
"supports_response_schema": false,
"supports_web_search": false,
"tpm": 250000
},
"gemini/gemini-3.5-transcribe": {
@ -58715,7 +58759,8 @@
],
"supports_audio_input": true,
"tpm": 800000,
"rpm": 2000
"rpm": 2000,
"supports_function_calling": false
},
"gemini/gemini-3.5-transcribe-live": {
"input_cost_per_audio_token": 3.5e-06,
@ -58735,7 +58780,8 @@
],
"supports_audio_input": true,
"tpm": 250000,
"rpm": 10
"rpm": 10,
"supports_function_calling": false
},
"vertex_ai/gemini-3.5-transcribe-preview": {
"input_cost_per_audio_token": 2e-06,
@ -61374,7 +61420,7 @@
"input_cost_per_audio_token": 1.5e-06,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "gemini",
"max_input_tokens": 131072,
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",

40
tests/test_litellm/integrations/otel/test_otel_v2_baggage.py
View file

@ -168,6 +168,46 @@ def test_allowlisted_metadata_subkey_promoted_blob_excluded():
assert all("private_note" not in k for k in span.attributes)
def test_nested_metadata_key_promoted_under_caller_path():
"""A dotted allowlist entry reads the nested caller metadata the proxy stores
under ``requester_metadata`` and lands on the LLM-call span under the caller's
own path (``litellm.metadata.trace_id``, ``litellm.metadata.nested.deep``);
a pre-existing flat dotted key keeps its full name, and unlisted siblings and
the blob stay out."""
engine, exporter = _engine_and_exporter()
payload = _payload()
payload["metadata"]["a.b"] = "flat"
payload["metadata"]["requester_metadata"] = {
"trace_id": "abc",
"attempt": 0,
"empty": "",
"nested": {"deep": "x", "skipped": "y"},
}
data = LLMCallSpanData.from_standard_logging_payload(payload)
bag = promoted_baggage(
data.identity,
data.request_model,
BAGGAGE_PROMOTED_KEYS,
metadata_keys=(
"requester_metadata.trace_id",
"requester_metadata.attempt",
"requester_metadata.empty",
"requester_metadata.nested.deep",
"a.b",
),
)
engine.emit(SpanRole.LLM_CALL, data, ctx_mod.set_request_baggage(bag))
(span,) = exporter.get_finished_spans()
assert span.attributes[f"{LiteLLM.METADATA_PREFIX}trace_id"] == "abc"
assert span.attributes[f"{LiteLLM.METADATA_PREFIX}attempt"] == "0"
assert span.attributes[f"{LiteLLM.METADATA_PREFIX}nested.deep"] == "x"
assert span.attributes[f"{LiteLLM.METADATA_PREFIX}a.b"] == "flat"
assert f"{LiteLLM.METADATA_PREFIX}empty" not in span.attributes
assert f"{LiteLLM.METADATA_PREFIX}deep" not in span.attributes
assert f"{LiteLLM.METADATA_PREFIX}nested.skipped" not in span.attributes
assert not any(k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") for k in span.attributes)
def test_http_attributes_never_promoted():
"""Even if http.* is present in baggage, the processor must not stamp it on
child spans (it belongs on the SERVER span only)."""

53
tests/test_litellm/integrations/otel/test_otel_v2_logger.py
View file

@ -1623,17 +1623,22 @@ def test_provider_model_and_team_metadata_on_real_boundary_flow():
def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans():
"""The pre-call hook seeds identity Baggage in the request context so the
server span (stamped directly) AND later child spans (service here, via the
Baggage processor) carry identity — not just the LLM-call span."""
Baggage processor) carry identity — not just the LLM-call span. Only the
caller's ``requester_metadata`` is read from the request dict, so a proxy-owned
sibling such as ``requester_ip_address`` is not stamped from here even though
the default allowlist names it, and an unlisted caller key is not promoted."""
logger, exporter = _logger()
server = logger._emitter.start_span(
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
)
data = {
"model": "gpt-4o",
"metadata": {"requester_ip_address": "127.0.0.1", "requester_metadata": {"trace_id": "abc"}},
}
async def _flow():
# pre-call seeds baggage + stamps the active server span
await logger.async_pre_call_hook(
_Auth(), None, {"model": "gpt-4o"}, "completion"
)
await logger.async_pre_call_hook(_Auth(), None, data, "completion")
# a later service call (same task) must inherit the identity
await logger.async_service_success_hook(
payload=_ServicePayload("redis", "set"), parent_otel_span=server
@ -1653,6 +1658,46 @@ def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans():
srv.attributes[LiteLLM.TEAM_ID] == "t1"
) # stamped directly on the server span
assert srv.attributes[f"{LiteLLM.METADATA_PREFIX}user_api_key_user_id"] == "u1"
assert not any(
k in (f"{LiteLLM.METADATA_PREFIX}requester_ip_address", f"{LiteLLM.METADATA_PREFIX}trace_id")
for s in (redis, srv)
for k in s.attributes
)
def test_pre_call_hook_promotes_nested_request_metadata_key():
"""``baggage_metadata_keys: [requester_metadata.trace_id]`` reads the caller's
``metadata.trace_id`` (snapshotted by the proxy under ``requester_metadata``)
and stamps ``litellm.metadata.trace_id`` on the server, LLM-call and service
spans of the request; unlisted siblings are not promoted."""
cfg = OpenTelemetryV2Config(exporter="in_memory", baggage_metadata_keys=["requester_metadata.trace_id"])
exporter = InMemorySpanExporter()
logger = OpenTelemetryV2(config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter))
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
data = {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}}
kwargs = _kwargs()
async def _flow():
await logger.async_pre_call_hook(_Auth(), None, data, "completion")
logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
await logger.async_log_success_event(kwargs, None, None, None)
await logger.async_service_success_hook(payload=_ServicePayload("redis", "set"), parent_otel_span=server)
with trace.use_span(server, end_on_exit=False):
asyncio.run(_flow())
server.end()
spans = {s.name: s for s in exporter.get_finished_spans()}
key = f"{LiteLLM.METADATA_PREFIX}trace_id"
assert spans[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes[key] == "abc"
assert spans["chat gpt-4o"].attributes[key] == "abc"
assert spans["redis set"].attributes[key] == "abc"
assert data == {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}}
assert not any(
k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") or k.endswith("deep")
for s in spans.values()
for k in s.attributes
)
# --------------------------------------------------------------------------- #

37
tests/test_litellm/integrations/test_opentelemetry.py
View file

@ -5582,6 +5582,38 @@ class TestOpenTelemetryInferenceIdentityAttributes(unittest.TestCase):
otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"})
assert "http.route" not in self._attr(span, exp)
def test_nested_metadata_key_promoted_under_caller_path(self):
"""``baggage_metadata_keys: [requester_metadata.trace_id]`` stamps the
caller's nested metadata value as ``litellm.metadata.trace_id`` and a deeper
path keeps its dotted name; unlisted siblings stay inside the
``metadata.requester_metadata`` blob."""
otel = OpenTelemetry(
config=OpenTelemetryConfig(
baggage_metadata_keys=["requester_metadata.trace_id", "requester_metadata.nested.deep"]
)
)
kwargs = self._kwargs()
kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = {
"trace_id": "abc",
"nested": {"deep": "x", "skipped": "y"},
}
span, exp = self._span()
otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"})
attrs = self._attr(span, exp)
assert attrs["litellm.metadata.trace_id"] == "abc"
assert attrs["litellm.metadata.nested.deep"] == "x"
assert "litellm.metadata.deep" not in attrs
assert "litellm.metadata.nested.skipped" not in attrs
assert not any(k.startswith("litellm.metadata.requester_metadata") for k in attrs)
def test_metadata_keys_default_to_none_promoted(self):
otel = OpenTelemetry()
kwargs = self._kwargs()
kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = {"trace_id": "abc"}
span, exp = self._span()
otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"})
assert not any(k.startswith("litellm.metadata.") for k in self._attr(span, exp))
def test_team_metadata_json_helper(self):
keys = ["a", "b"]
assert OpenTelemetry._team_metadata_json(None, keys) is None
@ -5632,6 +5664,11 @@ class TestOpenTelemetryTeamMetadataKeysConfig(unittest.TestCase):
cfg = OpenTelemetryConfig(baggage_team_metadata_keys=["from_arg"])
assert cfg.baggage_team_metadata_keys == ["from_arg"]
def test_metadata_keys_from_kwargs_and_env(self):
with patch.dict("os.environ", {"LITELLM_OTEL_BAGGAGE_METADATA_KEYS": "requester_metadata.trace_id, a.b"}):
assert OpenTelemetryConfig().baggage_metadata_keys == ["requester_metadata.trace_id", "a.b"]
assert OpenTelemetry(baggage_metadata_keys="x.y").config.baggage_metadata_keys == ["x.y"]
class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase):
"""LIT-3600: include/exclude control over which attributes are stamped on

43
tests/test_litellm/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py
View file

@ -1,3 +1,4 @@
from typing import Final
from unittest.mock import MagicMock
from litellm.llms.bedrock.vector_stores.transformation import BedrockVectorStoreConfig
@ -82,6 +83,7 @@ def test_transform_search_request_uses_only_retrieval_config_from_extra_body():
== "HYBRID"
)
assert "unrelatedField" not in body
assert "userContext" not in body
def test_transform_search_request_does_not_mutate_extra_body_and_overrides_number_of_results():
@ -152,3 +154,44 @@ def test_transform_search_request_overrides_filter_without_mutating_extra_body()
]["value"]
== "a"
)
def _search_body(extra_body: dict[str, object] | None, litellm_params: dict[str, object]) -> dict[str, object]:
config: Final = BedrockVectorStoreConfig()
mock_log: Final = MagicMock()
mock_log.model_call_details = {}
_, body = config.transform_search_vector_store_request(
vector_store_id="kb123",
query="hello",
vector_store_search_optional_params={"max_num_results": 3},
api_base="https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases",
litellm_logging_obj=mock_log,
litellm_params=litellm_params,
extra_body=extra_body,
)
return body
def test_transform_search_request_forwards_user_context_from_extra_body():
body = _search_body(extra_body={"userContext": {"userId": "alice@example.com"}}, litellm_params={})
assert body["userContext"] == {"userId": "alice@example.com"}
assert body["retrievalConfiguration"] == {"vectorSearchConfiguration": {"numberOfResults": 3}}
def test_transform_search_request_forwards_top_level_user_context_from_litellm_params():
body = _search_body(
extra_body=None,
litellm_params={"vector_store_id": "kb123", "user_context": {"userId": "bob@example.com"}},
)
assert body["userContext"] == {"userId": "bob@example.com"}
def test_transform_search_request_prefers_extra_body_user_context_over_top_level():
body = _search_body(
extra_body={"userContext": {"userId": "alice@example.com"}},
litellm_params={"userContext": {"userId": "bob@example.com"}},
)
assert body["userContext"] == {"userId": "alice@example.com"}

22
tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py
View file

@ -1189,6 +1189,28 @@ def test_reasoning_effort_integer_passthrough():
assert isinstance(result["reasoning_effort"], int)
def test_reasoning_effort_dict_from_anthropic_adapter_flattened_to_effort_string():
config = FireworksAIConfig()
result = config.map_openai_params(
{"reasoning_effort": {"effort": "medium", "summary": "detailed"}},
{},
_REASONING_MODEL,
drop_params=False,
)
assert result["reasoning_effort"] == "medium"
def test_reasoning_effort_dict_without_effort_key_dropped():
config = FireworksAIConfig()
result = config.map_openai_params(
{"reasoning_effort": {"summary": "detailed"}},
{},
_REASONING_MODEL,
drop_params=False,
)
assert "reasoning_effort" not in result
def test_reasoning_effort_auto_dropped_to_model_default():
config = FireworksAIConfig()
result = config.map_openai_params(

325
tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
View file

@ -28,6 +28,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
BaseOpenAIPassThroughHandler,
RouteChecks,
_join_url_paths,
anthropic_proxy_route,
azure_proxy_route,
bedrock_llm_proxy_route,
bedrock_proxy_route,
@ -585,6 +586,7 @@ class TestVertexAIPassThroughHandler:
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router",
pass_through_router,
)
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-master-1234")
endpoint = f"/v1/projects/{test_project}/locations/{test_location}/publishers/google/models/gemini-1.5-flash:generateContent"
@ -4286,6 +4288,329 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak:
assert "sk-master-1234" not in " ".join(f"{name}:{value}" for name, value in forwarded.items())
class TestAnthropicPassthroughVirtualKeyLeak:
VKEY = "sk-litellm-victim-key"
PROXY_KEY = "sk-ant-api03-proxy-configured-key"
ENDPOINT = "v1/messages"
async def _run(
self,
monkeypatch,
headers: list[tuple[bytes, bytes]],
authenticated: UserAPIKeyAuth | None = None,
master_key: str | None = "sk-master-1234",
proxy_api_key: str | None = None,
) -> tuple[HTTPException | None, dict | None]:
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import HttpPassThroughEndpointHelpers
from litellm.proxy.pass_through_endpoints.passthrough_endpoint_router import (
PassthroughEndpointRouter,
)
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", master_key)
monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False)
if proxy_api_key is None:
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
else:
monkeypatch.setenv("ANTHROPIC_API_KEY", proxy_api_key)
caller: Final = authenticated if authenticated is not None else UserAPIKeyAuth(api_key=self.VKEY)
async def receive():
return {"type": "http.request", "body": b"{}", "more_body": False}
request = Request(
{
"type": "http",
"method": "POST",
"path": f"/anthropic/{self.ENDPOINT}",
"headers": headers,
"query_string": b"",
},
receive=receive,
)
captured: dict = {}
def fake_create_pass_through_route(**kwargs):
captured.update(kwargs)
return AsyncMock(return_value={"status": "success"})
module = "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints"
monkeypatch.setattr(f"{module}.passthrough_endpoint_router", PassthroughEndpointRouter(lambda: None))
raised: HTTPException | None = None
with (
mock.patch(f"{module}.create_pass_through_route", side_effect=fake_create_pass_through_route),
mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value=caller)),
):
try:
await anthropic_proxy_route(
endpoint=self.ENDPOINT,
request=request,
fastapi_response=Response(),
user_api_key_dict=caller,
)
except HTTPException as exc:
raised = exc
if not captured:
return raised, None
upstream: Final = HttpPassThroughEndpointHelpers.forward_headers_from_request(
request_headers=dict(request.headers),
headers=dict(captured["custom_headers"] or {}),
forward_headers=captured.get("_forward_headers", False),
)
return raised, upstream
@staticmethod
def _blob(forwarded: dict) -> str:
return " ".join(f"{name}:{value}" for name, value in forwarded.items())
@pytest.mark.asyncio
async def test_authorization_bearer_virtual_key_is_rejected_not_forwarded(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[(b"authorization", f"Bearer {self.VKEY}".encode()), (b"content-type", b"application/json")],
)
assert forwarded is None, "credential-less request must never reach the upstream forwarder"
assert raised is not None and raised.status_code == 401
assert "ANTHROPIC_API_KEY" in str(raised.detail) and "use_in_pass_through" in str(raised.detail)
@pytest.mark.asyncio
async def test_x_api_key_virtual_key_is_rejected_not_forwarded(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[(b"x-api-key", self.VKEY.encode()), (b"content-type", b"application/json")],
)
assert forwarded is None, "a virtual key that authenticated via x-api-key must be stripped, not forwarded"
assert raised is not None and raised.status_code == 401
@pytest.mark.asyncio
async def test_x_litellm_api_key_virtual_key_is_rejected_not_forwarded(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[(b"x-litellm-api-key", self.VKEY.encode()), (b"content-type", b"application/json")],
)
assert forwarded is None, "credential-less request must never reach the upstream forwarder"
assert raised is not None and raised.status_code == 401
@pytest.mark.asyncio
async def test_master_key_in_authorization_is_rejected_not_forwarded(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[(b"authorization", b"Bearer sk-master-1234"), (b"content-type", b"application/json")],
authenticated=UserAPIKeyAuth(api_key="sk-master-1234", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert forwarded is None, "the master key must never reach Anthropic"
assert raised is not None and raised.status_code == 401
@pytest.mark.asyncio
@pytest.mark.parametrize(
("header", "value"),
[
pytest.param(b"x-api-key", b"sk-ant-api03-callers-own-key", id="x-api-key"),
pytest.param(b"authorization", b"Bearer sk-ant-api03-callers-own-key", id="authorization"),
],
)
async def test_without_a_master_key_the_callers_own_anthropic_key_still_forwards(
self, monkeypatch, header: bytes, value: bytes
):
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_auth", None)
raised, forwarded = await self._run(
monkeypatch,
[(header, value), (b"anthropic-version", b"2023-06-01"), (b"content-type", b"application/json")],
authenticated=UserAPIKeyAuth(api_key="sk-ant-api03-callers-own-key", user_role=LitellmUserRoles.INTERNAL_USER),
master_key=None,
)
assert raised is None, "with no master key the proxy authenticated nothing, so nothing of the caller's is a LiteLLM secret"
assert forwarded is not None
assert forwarded.get(header.decode()) == value.decode()
@pytest.mark.asyncio
async def test_without_a_master_key_a_custom_auth_credential_is_still_stripped(self, monkeypatch):
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_auth", AsyncMock())
raised, forwarded = await self._run(
monkeypatch,
[(b"authorization", b"Bearer sk-custom-auth-token"), (b"anthropic-version", b"2023-06-01")],
authenticated=UserAPIKeyAuth(api_key="sk-custom-auth-token", user_role=LitellmUserRoles.INTERNAL_USER),
master_key=None,
)
assert raised is not None and raised.status_code == 401
assert forwarded is None
@pytest.mark.asyncio
async def test_without_a_master_key_an_oauth2_token_is_still_stripped(self, monkeypatch):
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"enable_oauth2_auth": True})
monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_auth", None)
raised, forwarded = await self._run(
monkeypatch,
[(b"authorization", b"Bearer oauth2-access-token"), (b"anthropic-version", b"2023-06-01")],
authenticated=UserAPIKeyAuth(api_key="oauth2-access-token", user_role=LitellmUserRoles.INTERNAL_USER),
master_key=None,
)
assert raised is not None and raised.status_code == 401
assert forwarded is None
@pytest.mark.asyncio
async def test_byo_anthropic_oauth_token_still_forwards_without_virtual_key(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[
(b"x-litellm-api-key", self.VKEY.encode()),
(b"authorization", b"Bearer sk-ant-oat01-caller-oauth-token"),
(b"anthropic-version", b"2023-06-01"),
(b"content-type", b"application/json"),
],
)
assert raised is None
assert forwarded is not None
assert forwarded.get("authorization") == "Bearer sk-ant-oat01-caller-oauth-token"
assert forwarded.get("anthropic-version") == "2023-06-01"
assert "x-litellm-api-key" not in forwarded
assert self.VKEY not in self._blob(forwarded)
@pytest.mark.asyncio
async def test_byo_x_api_key_still_forwards_without_virtual_key(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[
(b"authorization", f"Bearer {self.VKEY}".encode()),
(b"x-api-key", b"sk-ant-api03-caller-own-key"),
(b"content-type", b"application/json"),
],
)
assert raised is None
assert forwarded is not None
assert forwarded.get("x-api-key") == "sk-ant-api03-caller-own-key"
assert "authorization" not in forwarded
assert self.VKEY not in self._blob(forwarded)
@pytest.mark.asyncio
async def test_custom_auth_caller_keeps_own_authorization_token(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[(b"authorization", b"Bearer sk-ant-oat01-caller-oauth-token"), (b"content-type", b"application/json")],
authenticated=UserAPIKeyAuth(api_key=None),
master_key=None,
)
assert raised is None
assert forwarded is not None
assert forwarded.get("authorization") == "Bearer sk-ant-oat01-caller-oauth-token"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"credential_header",
sorted(SpecialHeaders.litellm_credential_header_names() - {"authorization", "x-api-key", "x-litellm-api-key"}),
)
async def test_every_non_anthropic_credential_header_is_dropped_by_name(self, monkeypatch, credential_header):
raised, forwarded = await self._run(
monkeypatch,
[
(b"x-litellm-api-key", self.VKEY.encode()),
(b"x-api-key", b"sk-ant-api03-caller-own-key"),
(credential_header.encode(), b"some-distinct-caller-secret-value"),
(b"content-type", b"application/json"),
],
)
assert raised is None
assert forwarded is not None
assert forwarded.get("x-api-key") == "sk-ant-api03-caller-own-key"
assert credential_header not in forwarded
assert "x-litellm-api-key" not in forwarded
assert self.VKEY not in self._blob(forwarded)
assert "some-distinct-caller-secret-value" not in self._blob(forwarded)
@pytest.mark.asyncio
async def test_virtual_key_in_operator_configured_header_is_stripped(self, monkeypatch):
with mock.patch.dict( # test-quality-ok: general_settings is the real proxy config surface for litellm_key_header_name; no injection seam exists on this route
"litellm.proxy.proxy_server.general_settings",
{"litellm_key_header_name": "x-company-key"},
):
raised, forwarded = await self._run(
monkeypatch,
[
(b"x-company-key", f"Bearer {self.VKEY}".encode()),
(b"x-api-key", b"sk-ant-api03-caller-own-key"),
(b"content-type", b"application/json"),
],
)
assert raised is None
assert forwarded is not None
assert forwarded.get("x-api-key") == "sk-ant-api03-caller-own-key"
assert "x-company-key" not in forwarded
assert self.VKEY not in self._blob(forwarded)
@pytest.mark.asyncio
async def test_proxy_credential_replaces_virtual_key_sent_as_bearer(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[
(b"authorization", f"Bearer {self.VKEY}".encode()),
(b"anthropic-version", b"2023-06-01"),
(b"content-type", b"application/json"),
],
proxy_api_key=self.PROXY_KEY,
)
assert raised is None
assert forwarded is not None
assert forwarded.get("x-api-key") == self.PROXY_KEY
assert "authorization" not in forwarded
assert forwarded.get("anthropic-version") == "2023-06-01"
assert self.VKEY not in self._blob(forwarded)
@pytest.mark.asyncio
async def test_proxy_credential_replaces_virtual_key_sent_as_x_api_key(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[(b"x-api-key", self.VKEY.encode()), (b"content-type", b"application/json")],
proxy_api_key=self.PROXY_KEY,
)
assert raised is None
assert forwarded is not None
assert forwarded.get("x-api-key") == self.PROXY_KEY
assert self.VKEY not in self._blob(forwarded)
@pytest.mark.asyncio
async def test_proxy_credential_wins_over_callers_own_x_api_key(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[
(b"x-litellm-api-key", self.VKEY.encode()),
(b"x-api-key", b"sk-ant-api03-caller-own-key"),
(b"content-type", b"application/json"),
],
proxy_api_key=self.PROXY_KEY,
)
assert raised is None
assert forwarded is not None
assert forwarded.get("x-api-key") == self.PROXY_KEY
assert "sk-ant-api03-caller-own-key" not in self._blob(forwarded)
@pytest.mark.asyncio
async def test_x_pass_and_hop_by_hop_handling_is_unchanged(self, monkeypatch):
raised, forwarded = await self._run(
monkeypatch,
[
(b"authorization", f"Bearer {self.VKEY}".encode()),
(b"x-pass-anthropic-beta", b"interleaved-thinking-2025-05-14"),
(b"x-pass-authorization", b"Bearer smuggled"),
(b"content-length", b"2"),
(b"host", b"proxy.internal"),
(b"accept-encoding", b"br"),
(b"user-agent", b"curl/8.7.1"),
],
proxy_api_key=self.PROXY_KEY,
)
assert raised is None
assert forwarded is not None
assert forwarded.get("anthropic-beta") == "interleaved-thinking-2025-05-14"
assert forwarded.get("user-agent") == "curl/8.7.1"
assert "authorization" not in forwarded
assert "content-length" not in forwarded
assert "host" not in forwarded
assert "accept-encoding" not in forwarded
class TestVertexPassthroughDefaultLocationOnShortRoutes:
PROJECT = "test-project"
SHORT_ROUTE = "publishers/google/models/gemini-2.5-flash:generateContent"

43
tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py
View file

@ -752,6 +752,49 @@ def test_completed_event_restores_usage_hidden_by_stream_options_none():
assert completed.response.usage.output_tokens == 5
def _empty_choices_chunk(usage: Usage | None = None) -> ModelResponseStream:
return ModelResponseStream(id=CHAT_COMPLETION_ID, model="claude-haiku-4-5", choices=[], usage=usage)
@pytest.mark.asyncio
async def test_leading_empty_choices_chunk_does_not_kill_the_stream():
"""
Azure leads some streams with a `prompt_filter_results` chunk whose `choices` is empty.
The bridge used to index `choices[0]` on it and die before the first token.
"""
iterator = _build_iterator([_empty_choices_chunk(), _chunk("Hello"), _chunk("!", finish_reason="stop")])
events = [event async for event in iterator]
event_types = [getattr(event, "type", None) for event in events]
assert event_types.count(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) == 1
assert "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!"
assert event_types[-1] == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
@pytest.mark.asyncio
async def test_trailing_empty_choices_usage_chunk_reaches_response_completed():
"""
With `stream_options.include_usage` (which the bridge always sets) the last upstream chunk
carries only usage and an empty `choices`. It must not crash the stream, and its usage must
still land on `response.completed`.
"""
usage: Final = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
iterator = _build_iterator([_chunk("Hello"), _chunk("", finish_reason="stop"), _empty_choices_chunk(usage)])
events = [event async for event in iterator]
completed = next(
event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
)
assert completed.response.usage.input_tokens == 10
assert completed.response.usage.output_tokens == 5
def test_is_reasoning_end_ignores_empty_choices_chunk():
assert _build_iterator([])._is_reasoning_end(_empty_choices_chunk()) is False
def test_object_tool_call_arguments_stream_as_valid_json():
"""A provider that sends decoded object arguments must still stream valid JSON.

104
tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py
View file

@ -3,14 +3,7 @@ from pathlib import Path
import pytest
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.types.utils import (
ImageObject,
ImageResponse,
ImageUsage,
ImageUsageInputTokensDetails,
)
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
@ -21,94 +14,12 @@ GEMINI = "gemini/gemini-3.1-flash-lite-image"
VERTEX = "vertex_ai/gemini-3.1-flash-lite-image"
ALL_KEYS = (UNPREFIXED, GEMINI, VERTEX)
INPUT_COST = 2.5e-07
INPUT_COST_BATCHES = 1.25e-07
OUTPUT_TEXT_COST = 1.5e-06
OUTPUT_TEXT_COST_BATCHES = 7.5e-07
OUTPUT_IMAGE_TOKEN_COST = 3e-05
OUTPUT_COST_PER_1K_IMAGE = 0.0336
INPUT_COST_PER_IMAGE = 0.00028
CACHE_READ_COST = 2.5e-08
MAX_INPUT_TOKENS = 65536
MAX_OUTPUT_TOKENS = 4096
TOKENS_PER_1K_IMAGE = 1120
SHARED_FIELDS = {
"mode": "image_generation",
"input_cost_per_token": INPUT_COST,
"input_cost_per_token_batches": INPUT_COST_BATCHES,
"input_cost_per_image": INPUT_COST_PER_IMAGE,
"output_cost_per_token": OUTPUT_TEXT_COST,
"output_cost_per_token_batches": OUTPUT_TEXT_COST_BATCHES,
"output_cost_per_image": OUTPUT_COST_PER_1K_IMAGE,
"output_cost_per_image_token": OUTPUT_IMAGE_TOKEN_COST,
"max_input_tokens": MAX_INPUT_TOKENS,
"max_output_tokens": MAX_OUTPUT_TOKENS,
"max_tokens": MAX_OUTPUT_TOKENS,
"supported_endpoints": ["/v1/chat/completions", "/v1/completions", "/v1/batch"],
"supported_output_modalities": ["text", "image"],
"supports_reasoning": False,
"supports_response_schema": False,
"supports_system_messages": True,
"supports_vision": True,
}
VERTEX_ROUTE_FIELDS = {
"litellm_provider": "vertex_ai-language-models",
"cache_read_input_token_cost": CACHE_READ_COST,
"supported_modalities": ["text", "image", "video"],
"supports_function_calling": False,
"supports_pdf_input": True,
"supports_prompt_caching": True,
"supports_video_input": True,
}
PER_ROUTE_FIELDS = {
UNPREFIXED: VERTEX_ROUTE_FIELDS,
VERTEX: VERTEX_ROUTE_FIELDS,
GEMINI: {
"litellm_provider": "gemini",
"supported_modalities": ["text", "image"],
"supports_function_calling": True,
"supports_prompt_caching": False,
"rpm": 1000,
"tpm": 4000000,
},
}
GROUNDING_FIELDS = (
"supports_web_search",
"search_context_cost_per_query",
"web_search_billing_unit",
)
def _load(path: Path) -> dict:
with open(path, encoding="utf-8") as f:
return json.load(f)
@pytest.fixture
def local_model_cost_map(monkeypatch):
original_model_cost = litellm.model_cost
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.get_model_info.cache_clear()
try:
yield
finally:
litellm.model_cost = original_model_cost
litellm.get_model_info.cache_clear()
@pytest.mark.parametrize("model", ALL_KEYS)
@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup"))
def test_per_route_capabilities_match_model_cards(model: str, path: Path):
info = _load(path)[model]
for field, value in PER_ROUTE_FIELDS[model].items():
assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}"
@pytest.mark.parametrize("model", ALL_KEYS)
def test_backup_matches_main(model: str):
assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model)
@ -124,18 +35,3 @@ def test_vertex_prefix_routes_to_vertex():
routed_model, provider, _, _ = get_llm_provider(model=VERTEX)
assert routed_model == UNPREFIXED
assert provider == "vertex_ai"
def _one_k_image_response() -> ImageResponse:
return ImageResponse(
data=[ImageObject(b64_json="img1")],
usage=ImageUsage(
input_tokens=50 + TOKENS_PER_1K_IMAGE,
input_tokens_details=ImageUsageInputTokensDetails(
text_tokens=50,
image_tokens=TOKENS_PER_1K_IMAGE,
),
output_tokens=TOKENS_PER_1K_IMAGE,
total_tokens=50 + TOKENS_PER_1K_IMAGE + TOKENS_PER_1K_IMAGE,
),
)

25
tests/test_litellm/vector_stores/test_main.py
View file

@ -7,6 +7,7 @@ executor, and it must never leak into litellm_params/kwargs where logging would
model_dump() it (the #19550 serialization trap).
"""
import json
from unittest.mock import MagicMock, patch
import pytest
@ -15,6 +16,7 @@ import litellm.vector_stores.main as vector_stores_main
from litellm.llms.base_llm.vector_store.transformation import (
RouterVectorStoreEmbeddingExecutor,
)
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.vector_stores.main import search
MOCK_SEARCH_RESPONSE = {
@ -89,3 +91,26 @@ def test_search_router_not_in_litellm_params():
litellm_params = mock_handler.call_args.kwargs["litellm_params"]
assert "router" not in litellm_params.model_dump(exclude_none=True)
assert getattr(litellm_params, "router", None) is None
def test_search_forwards_top_level_user_context_to_bedrock_retrieve():
"""Regression (LIT-4415): a top-level userContext, the shape the OpenAI SDK's extra_body
produces on the proxy path, reaches the Bedrock Retrieve request body."""
client = MagicMock(spec=HTTPHandler)
client.post.return_value = MagicMock(status_code=200, json=MagicMock(return_value={"retrievalResults": []}))
search(
vector_store_id="kb123",
query="q",
custom_llm_provider="bedrock",
aws_region_name="us-west-2",
aws_access_key_id="test-key-id",
aws_secret_access_key="test-secret-key",
userContext={"userId": "alice@example.com"},
client=client,
litellm_logging_obj=MagicMock(),
)
posted = json.loads(client.post.call_args.kwargs["data"])
assert posted["userContext"] == {"userId": "alice@example.com"}
assert posted["retrievalQuery"] == {"text": "q"}