diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index c4847aca20d..551f783d4f9 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -70,7 +70,7 @@ env: jobs: rust-lint: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 defaults: run: working-directory: litellm-rust @@ -81,24 +81,48 @@ jobs: - run: rustup toolchain install --no-self-update - - run: cargo fmt --check + - run: cargo fmt --all --check - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - litellm-rust/target - key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-${{ github.job }}- + workspaces: litellm-rust + cache-on-failure: true - run: cargo clippy --workspace --all-targets --locked -- -D warnings rust-test: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 20 + defaults: + run: + working-directory: litellm-rust + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - run: rustup toolchain install --no-self-update + + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + with: + tool: cargo-nextest@0.9.143 + + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: litellm-rust + cache-on-failure: true + + - run: cargo nextest run --workspace --locked + + - run: cargo test --workspace --doc --locked + + rust-wheel: + runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: @@ -114,18 +138,10 @@ jobs: - run: rustup toolchain install --no-self-update - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - litellm-rust/target - key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-${{ github.job }}- - - - run: cargo test --workspace --locked - working-directory: litellm-rust + workspaces: litellm-rust + cache-on-failure: true - run: uv build --wheel --out-dir dist diff --git a/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs b/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs index cf9f31c3c13..c4b8d8eaae0 100644 --- a/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs +++ b/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs @@ -599,6 +599,34 @@ mod tests { static PYTHON_GLOBALS: Mutex<()> = Mutex::new(()); + fn install_lifecycle_module(py: Python<'_>) -> Bound<'_, PyModule> { + py.run( + pyo3::ffi::c_str!( + r#" +import sys +import types + +sys.modules.setdefault('litellm', types.ModuleType('litellm')) +sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bridge')) +"# + ), + None, + None, + ) + .unwrap(); + let source = std::ffi::CString::new(include_str!( + "../../../../../litellm/rust_bridge/lifecycle.py" + )) + .unwrap(); + PyModule::from_code( + py, + &source, + pyo3::ffi::c_str!("lifecycle.py"), + pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"), + ) + .unwrap() + } + fn install_logging_worker(py: Python<'_>, worker: &Bound<'_, PyAny>) -> PyResult<()> { py.import("litellm.litellm_core_utils.logging_worker")? .setattr("GLOBAL_LOGGING_WORKER", worker) @@ -773,17 +801,7 @@ mod tests { .unwrap_or_else(|error| error.into_inner()); Python::initialize(); Python::attach(|py| { - let source = std::ffi::CString::new(include_str!( - "../../../../../litellm/rust_bridge/lifecycle.py" - )) - .unwrap(); - PyModule::from_code( - py, - &source, - pyo3::ffi::c_str!("lifecycle.py"), - pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"), - ) - .unwrap(); + install_lifecycle_module(py); let route = SyntheticRoute( PythonCallState::new( py, @@ -819,17 +837,7 @@ mod tests { Python::initialize(); Python::attach(|py| { py.import("asyncio").unwrap(); - let source = std::ffi::CString::new(include_str!( - "../../../../../litellm/rust_bridge/lifecycle.py" - )) - .unwrap(); - let module = PyModule::from_code( - py, - &source, - pyo3::ffi::c_str!("lifecycle.py"), - pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"), - ) - .unwrap(); + let module = install_lifecycle_module(py); let locals = PyDict::new(py); locals .set_item("drive", module.getattr("drive").unwrap()) diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index 9038eb971b3..7f00298905f 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -190,6 +190,7 @@ mod tests { #[test] fn required_shapes_preserve_nested_values_and_existing_errors() { + Python::initialize(); let nested = json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]); assert_eq!( Value::Array(required_array("messages", nested.clone()).unwrap()), diff --git a/litellm-rust/crates/token-counter/src/tiktoken.rs b/litellm-rust/crates/token-counter/src/tiktoken.rs index c479ae01be9..7a9e71ed587 100644 --- a/litellm-rust/crates/token-counter/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter/src/tiktoken.rs @@ -195,14 +195,21 @@ mod tests { } #[test] - fn long_repeated_runs_stay_cheap() { + fn long_repeated_runs_cost_close_to_linear() { let ranks = ranks(); let mut scratch = MergeScratch::default(); - let piece = vec![b' '; 1 << 20]; - let started = std::time::Instant::now(); - let count = ranks.count_piece(&piece, &mut scratch); - assert!(count > 0); - assert!(started.elapsed().as_secs() < 5, "{:?}", started.elapsed()); + let mut time = |len: usize| { + let piece = vec![b' '; len]; + let started = std::time::Instant::now(); + assert!(ranks.count_piece(&piece, &mut scratch) > 0); + started.elapsed() + }; + let small = (0..3).map(|_| time(1 << 14)).min().unwrap(); + let large = time(1 << 18); + assert!( + large < small * 64, + "{small:?} for 2^14 bytes, {large:?} for 2^18" + ); } #[test] diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index afa6b795edd..34b712d47e3 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Iterable, Mapping from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, TypedDict, cast import litellm @@ -21,7 +22,9 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( parse_semconv_opt_in, ) from litellm.integrations.otel.mappers.utils import drop_none +from litellm.integrations.otel.model.baggage import promoted_metadata from litellm.integrations.otel.model.db_endpoint import db_span_attributes +from litellm.integrations.otel.model.metadata import flatten_metadata from litellm.integrations.otel.model.semconv import Metric from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -303,6 +306,7 @@ class OpenTelemetryConfig: # under ``litellm.team.metadata``. Empty by default so none of a team's # metadata leaves the process until explicitly allowlisted. baggage_team_metadata_keys: list[str] = field(default_factory=list) + baggage_metadata_keys: list[str] = field(default_factory=list) # Prometheus-style include/exclude control over which attributes are stamped # on emitted metrics, to cap metric cardinality. attributes: OTELMetricAttributeFilter | None = None @@ -329,6 +333,9 @@ class OpenTelemetryConfig: self.baggage_team_metadata_keys = _normalize_team_metadata_keys( self.baggage_team_metadata_keys ) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS")) + self.baggage_metadata_keys = _normalize_team_metadata_keys( + self.baggage_metadata_keys + ) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_METADATA_KEYS")) @classmethod def from_env(cls): @@ -381,11 +388,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): **kwargs, ): team_metadata_keys_override: Final = kwargs.pop("baggage_team_metadata_keys", None) + metadata_keys_override: Final = kwargs.pop("baggage_metadata_keys", None) metric_attributes_override: Final = kwargs.pop("attributes", None) if config is None: config = OpenTelemetryConfig.from_env() if team_metadata_keys_override is not None: config.baggage_team_metadata_keys = _normalize_team_metadata_keys(team_metadata_keys_override) + if metadata_keys_override is not None: + config.baggage_metadata_keys = _normalize_team_metadata_keys(metadata_keys_override) if metric_attributes_override is not None: config.attributes = _build_metric_attribute_filter(metric_attributes_override) @@ -1557,6 +1567,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if team_metadata: self.safe_set_attribute(span=span, key=TEAM_METADATA_ATTRIBUTE, value=team_metadata) + if self.config.baggage_metadata_keys: + flat_metadata: Final = MappingProxyType(dict(flatten_metadata(metadata))) + for key, value in promoted_metadata(flat_metadata, tuple(self.config.baggage_metadata_keys)).items(): + self.safe_set_attribute(span=span, key=key, value=value) + model_group: Final = standard_logging_payload.get("model_group") if model_group: self.safe_set_attribute(span=span, key=MODEL_GROUP_ATTRIBUTE, value=model_group) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 9ac748b231c..285a5c3aa97 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -33,6 +33,7 @@ from litellm.integrations.otel.model.metadata import ( LLMCallEvent, RequestIdentity, auth_metadata, + metadata_from_request_data, model_from_request_data, ) from litellm.integrations.otel.model.payloads import ( @@ -679,7 +680,12 @@ class OpenTelemetryV2(CustomLogger): # / errors are the FastAPI instrumentor's job, so we don't touch it here. # ====================================================================== # - def seed_request_identity(self, user_api_key_dict: object, model: str | None = None) -> None: + def seed_request_identity( + self, + user_api_key_dict: object, + model: str | None = None, + request_metadata: Mapping[str, object] | None = None, + ) -> None: """Attach request-identity Baggage to the current context + server span. Seeding identity into Baggage makes **every** span emitted afterwards for @@ -691,7 +697,7 @@ class OpenTelemetryV2(CustomLogger): isn't determined yet, which is correct. """ try: - identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict) + identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict, request_metadata) bag: Final = promoted_baggage( identity, model, @@ -743,6 +749,7 @@ class OpenTelemetryV2(CustomLogger): self.seed_request_identity( user_api_key_dict, model=model_from_request_data(data), + request_metadata=metadata_from_request_data(data), ) return data diff --git a/litellm/integrations/otel/model/baggage.py b/litellm/integrations/otel/model/baggage.py index 2be9bb36def..131848e1380 100644 --- a/litellm/integrations/otel/model/baggage.py +++ b/litellm/integrations/otel/model/baggage.py @@ -15,9 +15,10 @@ never promoted whole. import json from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import Final -from litellm.integrations.otel.model.metadata import RequestIdentity +from litellm.integrations.otel.model.metadata import REQUESTER_METADATA_PATH, RequestIdentity from litellm.integrations.otel.model.semconv import GenAI, LiteLLM # Attribute key -> value extractor over (identity, request_model, @@ -79,17 +80,23 @@ def promoted_baggage( ``team_metadata_keys`` selects sub-keys of the team's metadata to promote under ``litellm.team.metadata``. Empty values are dropped. """ - out: Final[dict[str, str]] = {} - for key, extract in _PROMOTABLE.items(): - if key in promoted_keys: - value = extract(identity, request_model, team_metadata_keys) - if value: - out[key] = value - for meta_key in metadata_keys: - value = identity.metadata.get(meta_key) - if value: - out[f"{LiteLLM.METADATA_PREFIX}{meta_key}"] = value - return out + identity_values: Final = { + key: value + for key, extract in _PROMOTABLE.items() + if key in promoted_keys and (value := extract(identity, request_model, team_metadata_keys)) + } + return {**identity_values, **promoted_metadata(identity.metadata, metadata_keys)} + + +def promoted_metadata(metadata: Mapping[str, str], metadata_keys: tuple[str, ...]) -> Mapping[str, str]: + """Allowlisted entries of a flattened metadata mapping under ``litellm.metadata.*``.""" + return MappingProxyType( + { + f"{LiteLLM.METADATA_PREFIX}{meta_key.removeprefix(REQUESTER_METADATA_PATH)}": value + for meta_key in metadata_keys + if (value := metadata.get(meta_key)) + } + ) def _filtered_team_metadata_json( diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index bd542ddc20c..5bda66ed618 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -210,7 +210,10 @@ class OpenTelemetryV2Config(BaseSettings): validation_alias=AliasChoices("baggage_metadata_keys", "LITELLM_OTEL_BAGGAGE_METADATA_KEYS"), description=( "Metadata sub-keys promoted under the ``litellm.metadata.*`` " - "namespace. Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` " + "namespace. A dotted path such as ``requester_metadata.trace_id`` " + "reads the caller's nested ``metadata.trace_id`` and is promoted as " + "``litellm.metadata.trace_id``; other dotted keys keep their full path. " + "Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` " "env var (comma-separated) or " "``callback_settings.otel.baggage_metadata_keys`` in config.yaml." ), diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index cc81b689708..5f90e70e119 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -49,6 +49,8 @@ if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload LANGFUSE_TRACE_NAME_HEADER: Final = "langfuse_trace_name" +REQUESTER_METADATA_KEY: Final = "requester_metadata" +REQUESTER_METADATA_PATH: Final = f"{REQUESTER_METADATA_KEY}." @dataclass(frozen=True) @@ -78,7 +80,7 @@ class RequestIdentity: model, not just the user-facing one. """ raw_meta: Final = cast(Mapping[str, object], payload.get("metadata") or {}) - metadata = {key: str(value) for key, value in raw_meta.items() if isinstance(value, (str, bool, int, float))} + metadata: Final = MappingProxyType(dict(flatten_metadata(raw_meta))) return cls( call_id=as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")), # StandardLoggingMetadata's canonical key is ``user_api_key_team_id``; @@ -95,7 +97,9 @@ class RequestIdentity: ) @classmethod - def from_user_api_key_auth(cls, auth: object) -> RequestIdentity: + def from_user_api_key_auth( + cls, auth: object, request_metadata: Mapping[str, object] | None = None + ) -> RequestIdentity: """Identity from a ``UserAPIKeyAuth`` (duck-typed to keep this module free of a proxy import). @@ -103,11 +107,13 @@ class RequestIdentity: guardrail, or service span is created — so the whole request's spans inherit identity, not just the LLM-call span. Metadata sub-keys use the ``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS`` - promotes. + promotes; ``request_metadata`` (the caller's ``requester_metadata`` + snapshot) is flattened to dotted keys so ``requester_metadata.`` + resolves too. """ get: Final = lambda name: getattr(auth, name, None) # noqa: E731 - metadata: Final = { - meta_key: str(value) + auth_meta: Final = tuple( + (meta_key, str(value)) for meta_key, attr in ( ("user_api_key_user_id", "user_id"), ("user_api_key_org_id", "org_id"), @@ -115,7 +121,9 @@ class RequestIdentity: ("user_api_key_end_user_id", "end_user_id"), ) if (value := get(attr)) - } + ) + request_meta: Final = flatten_metadata(request_metadata) if request_metadata is not None else () + metadata: Final = MappingProxyType(dict((*request_meta, *auth_meta))) return cls( team_id=as_str(get("team_id")), team_alias=as_str(get("team_alias")), @@ -351,6 +359,35 @@ def model_from_request_data(data: object) -> str | None: return None +def metadata_from_request_data(data: object) -> Mapping[str, object] | None: + """The caller's ``requester_metadata`` snapshot from a pre-call ``data`` dict, keyed under its wrapper. + + The proxy stores it under ``metadata`` or ``litellm_metadata`` depending on the route; + the proxy-owned siblings (``user_api_key_*``, ``requester_ip_address``) are not read. + """ + top: Final = _as_str_mapping(data) + if top is None: + return None + snapshots: Final = tuple( + snapshot + for name in ("metadata", "litellm_metadata") + if (nested := _as_str_mapping(top.get(name))) is not None + and (snapshot := _as_str_mapping(nested.get(REQUESTER_METADATA_KEY))) is not None + ) + return MappingProxyType({REQUESTER_METADATA_KEY: snapshots[0]}) if snapshots else None + + +def flatten_metadata(raw: Mapping[str, object]) -> Iterator[tuple[str, str]]: + """Scalar leaves of a nested metadata mapping, keyed by their dotted path.""" + stack: Final = list(tuple(raw.items())[::-1]) # mutable-ok: iterative worklist keeps the walk off the call stack + while stack: + key, value = stack.pop() + if (nested := _as_str_mapping(value)) is not None: + stack.extend(tuple((f"{key}.{sub_key}", sub_value) for sub_key, sub_value in nested.items())[::-1]) + elif isinstance(value, (str, bool, int, float)): + yield key, str(value) + + def resolve_provider_model(payload: StandardLoggingPayload) -> str | None: """The model litellm dispatched to the provider, from the payload. diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index 27c90c9d71e..ba8ce7e5625 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from copy import deepcopy from typing import TYPE_CHECKING, Any, Final, cast from urllib.parse import urlparse @@ -14,6 +15,7 @@ from litellm.types.integrations.rag.bedrock_knowledgebase import ( BedrockKBResponse, BedrockKBRetrievalConfiguration, BedrockKBRetrievalQuery, + BedrockKBUserContext, ) from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( @@ -242,10 +244,29 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): retrieval_config.setdefault("vectorSearchConfiguration", {})["filter"] = filters if retrieval_config: request_body["retrievalConfiguration"] = cast(BedrockKBRetrievalConfiguration, retrieval_config) + user_context: Final = self._user_context(extra_body=extra_body, litellm_params=litellm_params) + if user_context is not None: + request_body["userContext"] = user_context litellm_logging_obj.model_call_details["query"] = query return url, request_body + @staticmethod + def _user_context( + extra_body: Mapping[str, object] | None, litellm_params: Mapping[str, object] + ) -> BedrockKBUserContext | None: + sources: Final = tuple(source for source in (extra_body, litellm_params) if isinstance(source, Mapping)) + found: Final = next( + ( + source[key] + for source in sources + for key in ("userContext", "user_context") + if source.get(key) is not None + ), + None, + ) + return None if found is None else cast(BedrockKBUserContext, found) + def sign_request( self, headers: dict, diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 05160d83c12..b6c2b379d66 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -49,6 +49,17 @@ if TYPE_CHECKING: import tiktoken +def _map_reasoning_effort(value: object) -> object: + effort: Final[object] = cast(Mapping[str, object], value).get("effort") if isinstance(value, Mapping) else value + if effort is True: + return "medium" + if effort is False: + return "none" + if effort == "auto": + return None + return effort + + def _extract_fireworks_hidden_params(payload: dict) -> dict: """ Collect Fireworks-specific response fields (perf_metrics, prompt_token_ids, @@ -327,12 +338,9 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): elif param == "max_completion_tokens": optional_params["max_tokens"] = value elif param == "reasoning_effort": - if value is True: - optional_params["reasoning_effort"] = "medium" - elif value is False: - optional_params["reasoning_effort"] = "none" - elif value != "auto": - optional_params["reasoning_effort"] = value + effort = _map_reasoning_effort(value) + if effort is not None: + optional_params["reasoning_effort"] = effort elif param in supported_openai_params: if value is not None: optional_params[param] = value diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cd458666539..dd21bbf0b25 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -11313,13 +11313,16 @@ }, "azure_ai/grok-4.3": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 200000, "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, @@ -11331,13 +11334,16 @@ }, "azure_ai/grok-4.6": { "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, @@ -26162,7 +26168,9 @@ "output_vector_size": 3072, "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supports_audio_input": true, "supports_multimodal": true, + "supports_vision": true, "tpm": 10000000 }, "gemini/gemini-1.5-flash": { @@ -26303,8 +26311,8 @@ "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -26350,6 +26358,7 @@ "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, "output_cost_per_token_priority": 4.5e-06, + "supports_audio_input": true, "supports_image_size": false }, "gemini/gemini-2.5-flash-image": { @@ -26362,7 +26371,7 @@ "input_cost_per_token_priority": 5.4e-07, "litellm_provider": "gemini", "supports_reasoning": false, - "max_input_tokens": 32768, + "max_input_tokens": 65536, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "image_generation", @@ -26388,22 +26397,23 @@ "image" ], "supports_audio_output": false, - "supports_function_calling": true, + "supports_function_calling": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true, + "supports_web_search": false, "tpm": 8000000, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, + "supports_audio_input": false, "supports_image_size": false }, "gemini/gemini-3-pro-image": { @@ -26441,7 +26451,7 @@ ], "supports_function_calling": false, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, @@ -26544,7 +26554,7 @@ "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", - "max_input_tokens": 65536, + "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "image_generation", @@ -26571,7 +26581,7 @@ "supports_function_calling": false, "supports_prompt_caching": true, "supports_reasoning": false, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, @@ -26653,12 +26663,13 @@ "text", "image" ], - "supports_function_calling": true, + "supports_function_calling": false, "supports_prompt_caching": false, "supports_reasoning": false, "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, + "supports_web_search": false, "tpm": 4000000 }, "gemini/deep-research-pro-preview-12-2025": { @@ -26711,8 +26722,8 @@ "input_cost_per_token": 1e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -26758,6 +26769,7 @@ "output_cost_per_token_batches": 2e-07, "output_cost_per_token_flex": 2e-07, "output_cost_per_token_priority": 7.2e-07, + "supports_audio_input": true, "supports_image_size": false }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { @@ -27005,6 +27017,9 @@ "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, @@ -27013,7 +27028,11 @@ "/v1/audio/speech" ], "tpm": 4000000, - "rpm": 10 + "rpm": 10, + "supports_audio_input": false, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini/gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -27027,8 +27046,8 @@ "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -27338,8 +27357,8 @@ "input_cost_per_token": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -27388,7 +27407,8 @@ "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "output_cost_per_token_batches": 1.5e-06, - "output_cost_per_token_flex": 1.5e-06 + "output_cost_per_token_flex": 1.5e-06, + "supports_audio_input": true }, "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, @@ -27397,8 +27417,8 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -28112,9 +28132,9 @@ "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -28127,19 +28147,20 @@ "audio" ], "supports_audio_output": false, - "supports_function_calling": true, + "supports_function_calling": false, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, + "supports_vision": false, + "supports_web_search": false, "tpm": 10000000, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_audio_input": false }, "gemini/gemini-exp-1114": { "input_cost_per_token": 0, @@ -55917,7 +55938,7 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, + "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "realtime", @@ -55937,7 +55958,11 @@ ], "supports_audio_input": true, "supports_audio_output": true, - "gemini_native_audio": true + "gemini_native_audio": true, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true }, "gemini-3.1-flash-live-preview": { "input_cost_per_audio_token": 3e-06, @@ -55971,7 +55996,8 @@ "supports_vision": true, "supports_web_search": true, "gemini_audio_only_live": true, - "input_cost_per_second": 8.33333333333e-05 + "input_cost_per_second": 8.33333333333e-05, + "supports_response_schema": false }, "gemini/gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, @@ -56033,7 +56059,7 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, + "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "realtime", @@ -56055,7 +56081,11 @@ "supports_audio_output": true, "tpm": 250000, "rpm": 10, - "gemini_native_audio": true + "gemini_native_audio": true, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true }, "gemini/gemini-3.1-flash-live-preview": { "input_cost_per_audio_token": 3e-06, @@ -56091,7 +56121,8 @@ "tpm": 250000, "rpm": 10, "gemini_audio_only_live": true, - "input_cost_per_second": 8.33333333333e-05 + "input_cost_per_second": 8.33333333333e-05, + "supports_response_schema": false }, "gemini/gemini-3.1-flash-tts-preview": { "input_cost_per_token": 1e-06, @@ -56108,19 +56139,29 @@ "/v1/audio/speech" ], "tpm": 4000000, - "rpm": 10 + "rpm": 10, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/audio/speech" - ] + ], + "supports_audio_input": false, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini-flash-latest": { "cache_read_input_token_cost": 3e-08, @@ -58694,6 +58735,9 @@ ], "supports_audio_input": true, "supports_audio_output": true, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false, "tpm": 250000 }, "gemini/gemini-3.5-transcribe": { @@ -58715,7 +58759,8 @@ ], "supports_audio_input": true, "tpm": 800000, - "rpm": 2000 + "rpm": 2000, + "supports_function_calling": false }, "gemini/gemini-3.5-transcribe-live": { "input_cost_per_audio_token": 3.5e-06, @@ -58735,7 +58780,8 @@ ], "supports_audio_input": true, "tpm": 250000, - "rpm": 10 + "rpm": 10, + "supports_function_calling": false }, "vertex_ai/gemini-3.5-transcribe-preview": { "input_cost_per_audio_token": 2e-06, @@ -61374,7 +61420,7 @@ "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index ba267114bac..6757c0c594d 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2571,6 +2571,13 @@ def _token_can_vouch_for_team(valid_token: UserAPIKeyAuth, lookup_error: BaseExc return PrismaDBExceptionHandler.should_allow_request_on_db_unavailable() +def is_no_auth_dev_mode(master_key: str | None, general_settings: Mapping[str, object]) -> bool: + return master_key is None and not any( + general_settings.get(flag, False) + for flag in ("enable_jwt_auth", "enable_oauth2_auth", "enable_oauth2_proxy_auth") + ) + + @tracer.wrap() async def _run_centralized_common_checks( user_api_key_auth_obj: UserAPIKeyAuth, @@ -2630,11 +2637,7 @@ async def _run_centralized_common_checks( # Running common_checks would block every admin route on these # deployments where that was previously not the contract. If any # authn is enabled (JWT, OAuth2, OAuth2-proxy), authz must run. - if master_key is None and not ( - general_settings.get("enable_jwt_auth", False) - or general_settings.get("enable_oauth2_auth", False) - or general_settings.get("enable_oauth2_proxy_auth", False) - ): + if is_no_auth_dev_mode(master_key, general_settings): return if user_custom_auth is not None and not general_settings.get("custom_auth_run_common_checks", False): diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 955e6a8002b..0fe9d1cc626 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -45,6 +45,7 @@ from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( _get_bearer_token, + is_no_auth_dev_mode, user_api_key_auth, user_api_key_auth_websocket, ) @@ -709,8 +710,7 @@ async def anthropic_proxy_route( endpoint_func: Final = create_pass_through_route( endpoint=endpoint, target=str(updated_url), - custom_headers=auth_header if auth_header is not None else {}, - _forward_headers=True, + custom_headers=_upstream_headers_for_anthropic_route(request, user_api_key_dict, auth_header), is_streaming_request=is_streaming_request, ) # dynamically construct pass-through endpoint based on incoming path received_value: Final = await endpoint_func( @@ -1989,6 +1989,19 @@ _HEADERS_NEVER_FORWARDED_TO_VERTEX: Final = frozenset({"content-length", "host"} SpecialHeaders.litellm_credential_header_names() - _VERTEX_UPSTREAM_CREDENTIAL_HEADERS ) +_CREDENTIALLESS_ANTHROPIC_MISSING_CREDENTIAL_DETAIL: Final = ( + "No Anthropic credential is configured on this proxy and the request carried no upstream " + "Anthropic credential. The LiteLLM virtual key is not forwarded to Anthropic. Configure an " + "Anthropic credential (ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN, or a model with " + "use_in_pass_through: true), or send your own Anthropic API key in the x-api-key header or " + "your own Anthropic OAuth token in the Authorization header." +) + +_ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS: Final = frozenset({"authorization", "x-api-key"}) +_HEADERS_NEVER_FORWARDED_TO_ANTHROPIC: Final = frozenset({"content-length", "host", "accept-encoding"}) | ( + SpecialHeaders.litellm_credential_header_names() - _ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS +) + _MAPPED_ROUTE_CALLER_KEY_HEADER: Final = "litellm_user_api_key" @@ -2026,8 +2039,11 @@ def _is_authenticated_caller_jwt(value: str, jwt_claims: Mapping[str, object]) - def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAuth) -> bool: - """Whether a header value is the master key, the JWT that authenticated, or the key stored as ``api_key``.""" - from litellm.proxy.proxy_server import master_key + """Whether a header value is the master key, the JWT that authenticated, or the key stored as ``api_key``. + + A proxy in no-auth dev mode without custom auth authenticated nothing, so none of the caller's values is one. + """ + from litellm.proxy.proxy_server import general_settings, master_key, user_custom_auth normalized: Final = _normalize_credential_value(value) if master_key is not None and hmac.compare_digest(normalized.encode(), master_key.encode()): @@ -2035,35 +2051,54 @@ def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAut jwt_claims: Final = user_api_key_dict.jwt_claims if jwt_claims and _is_authenticated_caller_jwt(normalized, jwt_claims): return True + if is_no_auth_dev_mode(master_key, general_settings) and user_custom_auth is None: + return False authenticated_key: Final = user_api_key_dict.api_key if authenticated_key is None: return False - if master_key is None and not normalized.startswith("sk-"): - return False stored_representation: Final = UserAPIKeyAuth._safe_hash_litellm_api_key(normalized) # pyright: ignore[reportPrivateUsage] # the exact transform auth applied when it stored api_key return hmac.compare_digest(stored_representation.encode(), authenticated_key.encode()) +def _caller_headers_without_litellm_secrets( + request: Request, user_api_key_dict: UserAPIKeyAuth, never_forwarded: frozenset[str] +) -> Mapping[str, str]: + incoming: Final = _safe_get_request_headers(request) + dropped_by_name: Final = never_forwarded.union( + (_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names()) + ) + return MappingProxyType( + { + name: value + for name, value in incoming.items() + if name not in dropped_by_name and not _is_authenticated_caller_secret(value, user_api_key_dict) + } + ) + + def _forwarded_headers_for_credentialless_vertex_passthrough( request: Request, user_api_key_dict: UserAPIKeyAuth ) -> Mapping[str, str]: """Caller headers to forward on the bring-your-own-credentials Vertex branch, minus LiteLLM secrets.""" - incoming: Final = _safe_get_request_headers(request) - never_forwarded: Final = _HEADERS_NEVER_FORWARDED_TO_VERTEX.union( - (_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names()) + forwarded: Final = _caller_headers_without_litellm_secrets( + request, user_api_key_dict, _HEADERS_NEVER_FORWARDED_TO_VERTEX ) - forwarded: Final = MappingProxyType( - { - name: value - for name, value in incoming.items() - if name not in never_forwarded and not _is_authenticated_caller_secret(value, user_api_key_dict) - } - ) - if "authorization" not in forwarded and "x-goog-api-key" not in forwarded: + if _VERTEX_UPSTREAM_CREDENTIAL_HEADERS.isdisjoint(forwarded): raise HTTPException(status_code=401, detail=_CREDENTIALLESS_VERTEX_MISSING_CREDENTIAL_DETAIL) return forwarded +def _upstream_headers_for_anthropic_route( + request: Request, user_api_key_dict: UserAPIKeyAuth, proxy_auth_header: Mapping[str, str] | None +) -> Mapping[str, str]: + caller_headers: Final = _caller_headers_without_litellm_secrets( + request, user_api_key_dict, _HEADERS_NEVER_FORWARDED_TO_ANTHROPIC + ) + if proxy_auth_header is None and _ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS.isdisjoint(caller_headers): + raise HTTPException(status_code=401, detail=_CREDENTIALLESS_ANTHROPIC_MISSING_CREDENTIAL_DETAIL) + return MappingProxyType({**caller_headers, **(proxy_auth_header or {})}) + + async def _prepare_vertex_auth_headers( request: Request, vertex_credentials: VertexPassThroughCredentials | None, diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index c28b5558c75..1b9f39449cf 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -176,6 +176,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return {**item_kwargs, "name": tool_name, **namespace_kwargs} def _is_reasoning_end(self, chunk): + if not chunk.choices: + return False delta: Final = chunk.choices[0].delta # if this indicates reasoning content, don't consider reasoning ended @@ -897,6 +899,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): # Change: Never return a value, just enqueue output item events if self.sent_output_item_added_event: return + if not chunk.choices: + return delta: Final = chunk.choices[0].delta self._sequence_number += 1 @@ -1224,6 +1228,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): It's unclear how users expect litellm to translate multiple-choices-per-chunk to the responses API output. """ + if not choices: + return "" choice: Final = choices[0] chat_completion_delta: Final[ChatCompletionDelta] = choice.delta return chat_completion_delta.content or "" diff --git a/litellm/types/integrations/rag/bedrock_knowledgebase.py b/litellm/types/integrations/rag/bedrock_knowledgebase.py index e3aba85ed9b..7156d8101e1 100644 --- a/litellm/types/integrations/rag/bedrock_knowledgebase.py +++ b/litellm/types/integrations/rag/bedrock_knowledgebase.py @@ -1,6 +1,6 @@ from typing import Any, Literal -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class BedrockKBLocation(TypedDict, total=False): @@ -127,6 +127,10 @@ class BedrockKBGuardrailConfiguration(TypedDict, total=False): guardrailVersion: str | None +class BedrockKBUserContext(TypedDict): + userId: ReadOnly[str] + + class BedrockKBRequest(TypedDict, total=False): """Complete request structure for Bedrock Knowledge Base retrieval.""" @@ -134,6 +138,7 @@ class BedrockKBRequest(TypedDict, total=False): nextToken: str | None retrievalConfiguration: BedrockKBRetrievalConfiguration | None retrievalQuery: BedrockKBRetrievalQuery + userContext: ReadOnly[BedrockKBUserContext | None] ######################################################################### diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cd458666539..dd21bbf0b25 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -11313,13 +11313,16 @@ }, "azure_ai/grok-4.3": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 200000, "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, @@ -11331,13 +11334,16 @@ }, "azure_ai/grok-4.6": { "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, @@ -26162,7 +26168,9 @@ "output_vector_size": 3072, "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supports_audio_input": true, "supports_multimodal": true, + "supports_vision": true, "tpm": 10000000 }, "gemini/gemini-1.5-flash": { @@ -26303,8 +26311,8 @@ "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -26350,6 +26358,7 @@ "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, "output_cost_per_token_priority": 4.5e-06, + "supports_audio_input": true, "supports_image_size": false }, "gemini/gemini-2.5-flash-image": { @@ -26362,7 +26371,7 @@ "input_cost_per_token_priority": 5.4e-07, "litellm_provider": "gemini", "supports_reasoning": false, - "max_input_tokens": 32768, + "max_input_tokens": 65536, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "image_generation", @@ -26388,22 +26397,23 @@ "image" ], "supports_audio_output": false, - "supports_function_calling": true, + "supports_function_calling": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true, + "supports_web_search": false, "tpm": 8000000, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, + "supports_audio_input": false, "supports_image_size": false }, "gemini/gemini-3-pro-image": { @@ -26441,7 +26451,7 @@ ], "supports_function_calling": false, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, @@ -26544,7 +26554,7 @@ "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", - "max_input_tokens": 65536, + "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "image_generation", @@ -26571,7 +26581,7 @@ "supports_function_calling": false, "supports_prompt_caching": true, "supports_reasoning": false, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, @@ -26653,12 +26663,13 @@ "text", "image" ], - "supports_function_calling": true, + "supports_function_calling": false, "supports_prompt_caching": false, "supports_reasoning": false, "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, + "supports_web_search": false, "tpm": 4000000 }, "gemini/deep-research-pro-preview-12-2025": { @@ -26711,8 +26722,8 @@ "input_cost_per_token": 1e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -26758,6 +26769,7 @@ "output_cost_per_token_batches": 2e-07, "output_cost_per_token_flex": 2e-07, "output_cost_per_token_priority": 7.2e-07, + "supports_audio_input": true, "supports_image_size": false }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { @@ -27005,6 +27017,9 @@ "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, @@ -27013,7 +27028,11 @@ "/v1/audio/speech" ], "tpm": 4000000, - "rpm": 10 + "rpm": 10, + "supports_audio_input": false, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini/gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -27027,8 +27046,8 @@ "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -27338,8 +27357,8 @@ "input_cost_per_token": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -27388,7 +27407,8 @@ "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "output_cost_per_token_batches": 1.5e-06, - "output_cost_per_token_flex": 1.5e-06 + "output_cost_per_token_flex": 1.5e-06, + "supports_audio_input": true }, "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, @@ -27397,8 +27417,8 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -28112,9 +28132,9 @@ "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -28127,19 +28147,20 @@ "audio" ], "supports_audio_output": false, - "supports_function_calling": true, + "supports_function_calling": false, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, + "supports_vision": false, + "supports_web_search": false, "tpm": 10000000, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_audio_input": false }, "gemini/gemini-exp-1114": { "input_cost_per_token": 0, @@ -55917,7 +55938,7 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, + "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "realtime", @@ -55937,7 +55958,11 @@ ], "supports_audio_input": true, "supports_audio_output": true, - "gemini_native_audio": true + "gemini_native_audio": true, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true }, "gemini-3.1-flash-live-preview": { "input_cost_per_audio_token": 3e-06, @@ -55971,7 +55996,8 @@ "supports_vision": true, "supports_web_search": true, "gemini_audio_only_live": true, - "input_cost_per_second": 8.33333333333e-05 + "input_cost_per_second": 8.33333333333e-05, + "supports_response_schema": false }, "gemini/gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, @@ -56033,7 +56059,7 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, + "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "realtime", @@ -56055,7 +56081,11 @@ "supports_audio_output": true, "tpm": 250000, "rpm": 10, - "gemini_native_audio": true + "gemini_native_audio": true, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true }, "gemini/gemini-3.1-flash-live-preview": { "input_cost_per_audio_token": 3e-06, @@ -56091,7 +56121,8 @@ "tpm": 250000, "rpm": 10, "gemini_audio_only_live": true, - "input_cost_per_second": 8.33333333333e-05 + "input_cost_per_second": 8.33333333333e-05, + "supports_response_schema": false }, "gemini/gemini-3.1-flash-tts-preview": { "input_cost_per_token": 1e-06, @@ -56108,19 +56139,29 @@ "/v1/audio/speech" ], "tpm": 4000000, - "rpm": 10 + "rpm": 10, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/audio/speech" - ] + ], + "supports_audio_input": false, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini-flash-latest": { "cache_read_input_token_cost": 3e-08, @@ -58694,6 +58735,9 @@ ], "supports_audio_input": true, "supports_audio_output": true, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false, "tpm": 250000 }, "gemini/gemini-3.5-transcribe": { @@ -58715,7 +58759,8 @@ ], "supports_audio_input": true, "tpm": 800000, - "rpm": 2000 + "rpm": 2000, + "supports_function_calling": false }, "gemini/gemini-3.5-transcribe-live": { "input_cost_per_audio_token": 3.5e-06, @@ -58735,7 +58780,8 @@ ], "supports_audio_input": true, "tpm": 250000, - "rpm": 10 + "rpm": 10, + "supports_function_calling": false }, "vertex_ai/gemini-3.5-transcribe-preview": { "input_cost_per_audio_token": 2e-06, @@ -61374,7 +61420,7 @@ "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py b/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py index b379b8bebc9..930c01e524e 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py @@ -168,6 +168,46 @@ def test_allowlisted_metadata_subkey_promoted_blob_excluded(): assert all("private_note" not in k for k in span.attributes) +def test_nested_metadata_key_promoted_under_caller_path(): + """A dotted allowlist entry reads the nested caller metadata the proxy stores + under ``requester_metadata`` and lands on the LLM-call span under the caller's + own path (``litellm.metadata.trace_id``, ``litellm.metadata.nested.deep``); + a pre-existing flat dotted key keeps its full name, and unlisted siblings and + the blob stay out.""" + engine, exporter = _engine_and_exporter() + payload = _payload() + payload["metadata"]["a.b"] = "flat" + payload["metadata"]["requester_metadata"] = { + "trace_id": "abc", + "attempt": 0, + "empty": "", + "nested": {"deep": "x", "skipped": "y"}, + } + data = LLMCallSpanData.from_standard_logging_payload(payload) + bag = promoted_baggage( + data.identity, + data.request_model, + BAGGAGE_PROMOTED_KEYS, + metadata_keys=( + "requester_metadata.trace_id", + "requester_metadata.attempt", + "requester_metadata.empty", + "requester_metadata.nested.deep", + "a.b", + ), + ) + engine.emit(SpanRole.LLM_CALL, data, ctx_mod.set_request_baggage(bag)) + (span,) = exporter.get_finished_spans() + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}trace_id"] == "abc" + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}attempt"] == "0" + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}nested.deep"] == "x" + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}a.b"] == "flat" + assert f"{LiteLLM.METADATA_PREFIX}empty" not in span.attributes + assert f"{LiteLLM.METADATA_PREFIX}deep" not in span.attributes + assert f"{LiteLLM.METADATA_PREFIX}nested.skipped" not in span.attributes + assert not any(k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") for k in span.attributes) + + def test_http_attributes_never_promoted(): """Even if http.* is present in baggage, the processor must not stamp it on child spans (it belongs on the SERVER span only).""" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 2869c804c07..9b5abae60cc 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -1623,17 +1623,22 @@ def test_provider_model_and_team_metadata_on_real_boundary_flow(): def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans(): """The pre-call hook seeds identity Baggage in the request context so the server span (stamped directly) AND later child spans (service here, via the - Baggage processor) carry identity — not just the LLM-call span.""" + Baggage processor) carry identity — not just the LLM-call span. Only the + caller's ``requester_metadata`` is read from the request dict, so a proxy-owned + sibling such as ``requester_ip_address`` is not stamped from here even though + the default allowlist names it, and an unlisted caller key is not promoted.""" logger, exporter = _logger() server = logger._emitter.start_span( SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME ) + data = { + "model": "gpt-4o", + "metadata": {"requester_ip_address": "127.0.0.1", "requester_metadata": {"trace_id": "abc"}}, + } async def _flow(): # pre-call seeds baggage + stamps the active server span - await logger.async_pre_call_hook( - _Auth(), None, {"model": "gpt-4o"}, "completion" - ) + await logger.async_pre_call_hook(_Auth(), None, data, "completion") # a later service call (same task) must inherit the identity await logger.async_service_success_hook( payload=_ServicePayload("redis", "set"), parent_otel_span=server @@ -1653,6 +1658,46 @@ def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans(): srv.attributes[LiteLLM.TEAM_ID] == "t1" ) # stamped directly on the server span assert srv.attributes[f"{LiteLLM.METADATA_PREFIX}user_api_key_user_id"] == "u1" + assert not any( + k in (f"{LiteLLM.METADATA_PREFIX}requester_ip_address", f"{LiteLLM.METADATA_PREFIX}trace_id") + for s in (redis, srv) + for k in s.attributes + ) + + +def test_pre_call_hook_promotes_nested_request_metadata_key(): + """``baggage_metadata_keys: [requester_metadata.trace_id]`` reads the caller's + ``metadata.trace_id`` (snapshotted by the proxy under ``requester_metadata``) + and stamps ``litellm.metadata.trace_id`` on the server, LLM-call and service + spans of the request; unlisted siblings are not promoted.""" + cfg = OpenTelemetryV2Config(exporter="in_memory", baggage_metadata_keys=["requester_metadata.trace_id"]) + exporter = InMemorySpanExporter() + logger = OpenTelemetryV2(config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter)) + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + data = {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}} + kwargs = _kwargs() + + async def _flow(): + await logger.async_pre_call_hook(_Auth(), None, data, "completion") + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + await logger.async_log_success_event(kwargs, None, None, None) + await logger.async_service_success_hook(payload=_ServicePayload("redis", "set"), parent_otel_span=server) + + with trace.use_span(server, end_on_exit=False): + asyncio.run(_flow()) + server.end() + + spans = {s.name: s for s in exporter.get_finished_spans()} + key = f"{LiteLLM.METADATA_PREFIX}trace_id" + assert spans[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes[key] == "abc" + assert spans["chat gpt-4o"].attributes[key] == "abc" + assert spans["redis set"].attributes[key] == "abc" + assert data == {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}} + assert not any( + k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") or k.endswith("deep") + for s in spans.values() + for k in s.attributes + ) # --------------------------------------------------------------------------- # diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 8a8c076fe2e..7d1faf93116 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -5582,6 +5582,38 @@ class TestOpenTelemetryInferenceIdentityAttributes(unittest.TestCase): otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"}) assert "http.route" not in self._attr(span, exp) + def test_nested_metadata_key_promoted_under_caller_path(self): + """``baggage_metadata_keys: [requester_metadata.trace_id]`` stamps the + caller's nested metadata value as ``litellm.metadata.trace_id`` and a deeper + path keeps its dotted name; unlisted siblings stay inside the + ``metadata.requester_metadata`` blob.""" + otel = OpenTelemetry( + config=OpenTelemetryConfig( + baggage_metadata_keys=["requester_metadata.trace_id", "requester_metadata.nested.deep"] + ) + ) + kwargs = self._kwargs() + kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = { + "trace_id": "abc", + "nested": {"deep": "x", "skipped": "y"}, + } + span, exp = self._span() + otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"}) + attrs = self._attr(span, exp) + assert attrs["litellm.metadata.trace_id"] == "abc" + assert attrs["litellm.metadata.nested.deep"] == "x" + assert "litellm.metadata.deep" not in attrs + assert "litellm.metadata.nested.skipped" not in attrs + assert not any(k.startswith("litellm.metadata.requester_metadata") for k in attrs) + + def test_metadata_keys_default_to_none_promoted(self): + otel = OpenTelemetry() + kwargs = self._kwargs() + kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = {"trace_id": "abc"} + span, exp = self._span() + otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"}) + assert not any(k.startswith("litellm.metadata.") for k in self._attr(span, exp)) + def test_team_metadata_json_helper(self): keys = ["a", "b"] assert OpenTelemetry._team_metadata_json(None, keys) is None @@ -5632,6 +5664,11 @@ class TestOpenTelemetryTeamMetadataKeysConfig(unittest.TestCase): cfg = OpenTelemetryConfig(baggage_team_metadata_keys=["from_arg"]) assert cfg.baggage_team_metadata_keys == ["from_arg"] + def test_metadata_keys_from_kwargs_and_env(self): + with patch.dict("os.environ", {"LITELLM_OTEL_BAGGAGE_METADATA_KEYS": "requester_metadata.trace_id, a.b"}): + assert OpenTelemetryConfig().baggage_metadata_keys == ["requester_metadata.trace_id", "a.b"] + assert OpenTelemetry(baggage_metadata_keys="x.y").config.baggage_metadata_keys == ["x.y"] + class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): """LIT-3600: include/exclude control over which attributes are stamped on diff --git a/tests/test_litellm/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py b/tests/test_litellm/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py index 7b04efa17dc..ab5a2531461 100644 --- a/tests/test_litellm/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py +++ b/tests/test_litellm/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py @@ -1,3 +1,4 @@ +from typing import Final from unittest.mock import MagicMock from litellm.llms.bedrock.vector_stores.transformation import BedrockVectorStoreConfig @@ -82,6 +83,7 @@ def test_transform_search_request_uses_only_retrieval_config_from_extra_body(): == "HYBRID" ) assert "unrelatedField" not in body + assert "userContext" not in body def test_transform_search_request_does_not_mutate_extra_body_and_overrides_number_of_results(): @@ -152,3 +154,44 @@ def test_transform_search_request_overrides_filter_without_mutating_extra_body() ]["value"] == "a" ) + + +def _search_body(extra_body: dict[str, object] | None, litellm_params: dict[str, object]) -> dict[str, object]: + config: Final = BedrockVectorStoreConfig() + mock_log: Final = MagicMock() + mock_log.model_call_details = {} + _, body = config.transform_search_vector_store_request( + vector_store_id="kb123", + query="hello", + vector_store_search_optional_params={"max_num_results": 3}, + api_base="https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases", + litellm_logging_obj=mock_log, + litellm_params=litellm_params, + extra_body=extra_body, + ) + return body + + +def test_transform_search_request_forwards_user_context_from_extra_body(): + body = _search_body(extra_body={"userContext": {"userId": "alice@example.com"}}, litellm_params={}) + + assert body["userContext"] == {"userId": "alice@example.com"} + assert body["retrievalConfiguration"] == {"vectorSearchConfiguration": {"numberOfResults": 3}} + + +def test_transform_search_request_forwards_top_level_user_context_from_litellm_params(): + body = _search_body( + extra_body=None, + litellm_params={"vector_store_id": "kb123", "user_context": {"userId": "bob@example.com"}}, + ) + + assert body["userContext"] == {"userId": "bob@example.com"} + + +def test_transform_search_request_prefers_extra_body_user_context_over_top_level(): + body = _search_body( + extra_body={"userContext": {"userId": "alice@example.com"}}, + litellm_params={"userContext": {"userId": "bob@example.com"}}, + ) + + assert body["userContext"] == {"userId": "alice@example.com"} diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index fb0311ef39b..7715e7b32ff 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1189,6 +1189,28 @@ def test_reasoning_effort_integer_passthrough(): assert isinstance(result["reasoning_effort"], int) +def test_reasoning_effort_dict_from_anthropic_adapter_flattened_to_effort_string(): + config = FireworksAIConfig() + result = config.map_openai_params( + {"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, + {}, + _REASONING_MODEL, + drop_params=False, + ) + assert result["reasoning_effort"] == "medium" + + +def test_reasoning_effort_dict_without_effort_key_dropped(): + config = FireworksAIConfig() + result = config.map_openai_params( + {"reasoning_effort": {"summary": "detailed"}}, + {}, + _REASONING_MODEL, + drop_params=False, + ) + assert "reasoning_effort" not in result + + def test_reasoning_effort_auto_dropped_to_model_default(): config = FireworksAIConfig() result = config.map_openai_params( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 73e6ceabdb6..e0785b002b2 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -28,6 +28,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, RouteChecks, _join_url_paths, + anthropic_proxy_route, azure_proxy_route, bedrock_llm_proxy_route, bedrock_proxy_route, @@ -585,6 +586,7 @@ class TestVertexAIPassThroughHandler: "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router", pass_through_router, ) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-master-1234") endpoint = f"/v1/projects/{test_project}/locations/{test_location}/publishers/google/models/gemini-1.5-flash:generateContent" @@ -4286,6 +4288,329 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: assert "sk-master-1234" not in " ".join(f"{name}:{value}" for name, value in forwarded.items()) +class TestAnthropicPassthroughVirtualKeyLeak: + VKEY = "sk-litellm-victim-key" + PROXY_KEY = "sk-ant-api03-proxy-configured-key" + ENDPOINT = "v1/messages" + + async def _run( + self, + monkeypatch, + headers: list[tuple[bytes, bytes]], + authenticated: UserAPIKeyAuth | None = None, + master_key: str | None = "sk-master-1234", + proxy_api_key: str | None = None, + ) -> tuple[HTTPException | None, dict | None]: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import HttpPassThroughEndpointHelpers + from litellm.proxy.pass_through_endpoints.passthrough_endpoint_router import ( + PassthroughEndpointRouter, + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", master_key) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + if proxy_api_key is None: + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + else: + monkeypatch.setenv("ANTHROPIC_API_KEY", proxy_api_key) + caller: Final = authenticated if authenticated is not None else UserAPIKeyAuth(api_key=self.VKEY) + + async def receive(): + return {"type": "http.request", "body": b"{}", "more_body": False} + + request = Request( + { + "type": "http", + "method": "POST", + "path": f"/anthropic/{self.ENDPOINT}", + "headers": headers, + "query_string": b"", + }, + receive=receive, + ) + + captured: dict = {} + + def fake_create_pass_through_route(**kwargs): + captured.update(kwargs) + return AsyncMock(return_value={"status": "success"}) + + module = "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints" + monkeypatch.setattr(f"{module}.passthrough_endpoint_router", PassthroughEndpointRouter(lambda: None)) + raised: HTTPException | None = None + with ( + mock.patch(f"{module}.create_pass_through_route", side_effect=fake_create_pass_through_route), + mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value=caller)), + ): + try: + await anthropic_proxy_route( + endpoint=self.ENDPOINT, + request=request, + fastapi_response=Response(), + user_api_key_dict=caller, + ) + except HTTPException as exc: + raised = exc + + if not captured: + return raised, None + upstream: Final = HttpPassThroughEndpointHelpers.forward_headers_from_request( + request_headers=dict(request.headers), + headers=dict(captured["custom_headers"] or {}), + forward_headers=captured.get("_forward_headers", False), + ) + return raised, upstream + + @staticmethod + def _blob(forwarded: dict) -> str: + return " ".join(f"{name}:{value}" for name, value in forwarded.items()) + + @pytest.mark.asyncio + async def test_authorization_bearer_virtual_key_is_rejected_not_forwarded(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", f"Bearer {self.VKEY}".encode()), (b"content-type", b"application/json")], + ) + assert forwarded is None, "credential-less request must never reach the upstream forwarder" + assert raised is not None and raised.status_code == 401 + assert "ANTHROPIC_API_KEY" in str(raised.detail) and "use_in_pass_through" in str(raised.detail) + + @pytest.mark.asyncio + async def test_x_api_key_virtual_key_is_rejected_not_forwarded(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"x-api-key", self.VKEY.encode()), (b"content-type", b"application/json")], + ) + assert forwarded is None, "a virtual key that authenticated via x-api-key must be stripped, not forwarded" + assert raised is not None and raised.status_code == 401 + + @pytest.mark.asyncio + async def test_x_litellm_api_key_virtual_key_is_rejected_not_forwarded(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"x-litellm-api-key", self.VKEY.encode()), (b"content-type", b"application/json")], + ) + assert forwarded is None, "credential-less request must never reach the upstream forwarder" + assert raised is not None and raised.status_code == 401 + + @pytest.mark.asyncio + async def test_master_key_in_authorization_is_rejected_not_forwarded(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", b"Bearer sk-master-1234"), (b"content-type", b"application/json")], + authenticated=UserAPIKeyAuth(api_key="sk-master-1234", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert forwarded is None, "the master key must never reach Anthropic" + assert raised is not None and raised.status_code == 401 + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("header", "value"), + [ + pytest.param(b"x-api-key", b"sk-ant-api03-callers-own-key", id="x-api-key"), + pytest.param(b"authorization", b"Bearer sk-ant-api03-callers-own-key", id="authorization"), + ], + ) + async def test_without_a_master_key_the_callers_own_anthropic_key_still_forwards( + self, monkeypatch, header: bytes, value: bytes + ): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_auth", None) + raised, forwarded = await self._run( + monkeypatch, + [(header, value), (b"anthropic-version", b"2023-06-01"), (b"content-type", b"application/json")], + authenticated=UserAPIKeyAuth(api_key="sk-ant-api03-callers-own-key", user_role=LitellmUserRoles.INTERNAL_USER), + master_key=None, + ) + assert raised is None, "with no master key the proxy authenticated nothing, so nothing of the caller's is a LiteLLM secret" + assert forwarded is not None + assert forwarded.get(header.decode()) == value.decode() + + @pytest.mark.asyncio + async def test_without_a_master_key_a_custom_auth_credential_is_still_stripped(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_auth", AsyncMock()) + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", b"Bearer sk-custom-auth-token"), (b"anthropic-version", b"2023-06-01")], + authenticated=UserAPIKeyAuth(api_key="sk-custom-auth-token", user_role=LitellmUserRoles.INTERNAL_USER), + master_key=None, + ) + assert raised is not None and raised.status_code == 401 + assert forwarded is None + + @pytest.mark.asyncio + async def test_without_a_master_key_an_oauth2_token_is_still_stripped(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"enable_oauth2_auth": True}) + monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_auth", None) + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", b"Bearer oauth2-access-token"), (b"anthropic-version", b"2023-06-01")], + authenticated=UserAPIKeyAuth(api_key="oauth2-access-token", user_role=LitellmUserRoles.INTERNAL_USER), + master_key=None, + ) + assert raised is not None and raised.status_code == 401 + assert forwarded is None + + @pytest.mark.asyncio + async def test_byo_anthropic_oauth_token_still_forwards_without_virtual_key(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"x-litellm-api-key", self.VKEY.encode()), + (b"authorization", b"Bearer sk-ant-oat01-caller-oauth-token"), + (b"anthropic-version", b"2023-06-01"), + (b"content-type", b"application/json"), + ], + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("authorization") == "Bearer sk-ant-oat01-caller-oauth-token" + assert forwarded.get("anthropic-version") == "2023-06-01" + assert "x-litellm-api-key" not in forwarded + assert self.VKEY not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_byo_x_api_key_still_forwards_without_virtual_key(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"authorization", f"Bearer {self.VKEY}".encode()), + (b"x-api-key", b"sk-ant-api03-caller-own-key"), + (b"content-type", b"application/json"), + ], + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == "sk-ant-api03-caller-own-key" + assert "authorization" not in forwarded + assert self.VKEY not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_custom_auth_caller_keeps_own_authorization_token(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", b"Bearer sk-ant-oat01-caller-oauth-token"), (b"content-type", b"application/json")], + authenticated=UserAPIKeyAuth(api_key=None), + master_key=None, + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("authorization") == "Bearer sk-ant-oat01-caller-oauth-token" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "credential_header", + sorted(SpecialHeaders.litellm_credential_header_names() - {"authorization", "x-api-key", "x-litellm-api-key"}), + ) + async def test_every_non_anthropic_credential_header_is_dropped_by_name(self, monkeypatch, credential_header): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"x-litellm-api-key", self.VKEY.encode()), + (b"x-api-key", b"sk-ant-api03-caller-own-key"), + (credential_header.encode(), b"some-distinct-caller-secret-value"), + (b"content-type", b"application/json"), + ], + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == "sk-ant-api03-caller-own-key" + assert credential_header not in forwarded + assert "x-litellm-api-key" not in forwarded + assert self.VKEY not in self._blob(forwarded) + assert "some-distinct-caller-secret-value" not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_virtual_key_in_operator_configured_header_is_stripped(self, monkeypatch): + with mock.patch.dict( # test-quality-ok: general_settings is the real proxy config surface for litellm_key_header_name; no injection seam exists on this route + "litellm.proxy.proxy_server.general_settings", + {"litellm_key_header_name": "x-company-key"}, + ): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"x-company-key", f"Bearer {self.VKEY}".encode()), + (b"x-api-key", b"sk-ant-api03-caller-own-key"), + (b"content-type", b"application/json"), + ], + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == "sk-ant-api03-caller-own-key" + assert "x-company-key" not in forwarded + assert self.VKEY not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_proxy_credential_replaces_virtual_key_sent_as_bearer(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"authorization", f"Bearer {self.VKEY}".encode()), + (b"anthropic-version", b"2023-06-01"), + (b"content-type", b"application/json"), + ], + proxy_api_key=self.PROXY_KEY, + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == self.PROXY_KEY + assert "authorization" not in forwarded + assert forwarded.get("anthropic-version") == "2023-06-01" + assert self.VKEY not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_proxy_credential_replaces_virtual_key_sent_as_x_api_key(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"x-api-key", self.VKEY.encode()), (b"content-type", b"application/json")], + proxy_api_key=self.PROXY_KEY, + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == self.PROXY_KEY + assert self.VKEY not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_proxy_credential_wins_over_callers_own_x_api_key(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"x-litellm-api-key", self.VKEY.encode()), + (b"x-api-key", b"sk-ant-api03-caller-own-key"), + (b"content-type", b"application/json"), + ], + proxy_api_key=self.PROXY_KEY, + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == self.PROXY_KEY + assert "sk-ant-api03-caller-own-key" not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_x_pass_and_hop_by_hop_handling_is_unchanged(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"authorization", f"Bearer {self.VKEY}".encode()), + (b"x-pass-anthropic-beta", b"interleaved-thinking-2025-05-14"), + (b"x-pass-authorization", b"Bearer smuggled"), + (b"content-length", b"2"), + (b"host", b"proxy.internal"), + (b"accept-encoding", b"br"), + (b"user-agent", b"curl/8.7.1"), + ], + proxy_api_key=self.PROXY_KEY, + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("anthropic-beta") == "interleaved-thinking-2025-05-14" + assert forwarded.get("user-agent") == "curl/8.7.1" + assert "authorization" not in forwarded + assert "content-length" not in forwarded + assert "host" not in forwarded + assert "accept-encoding" not in forwarded + + class TestVertexPassthroughDefaultLocationOnShortRoutes: PROJECT = "test-project" SHORT_ROUTE = "publishers/google/models/gemini-2.5-flash:generateContent" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 5d97b0531d6..343fc873fa4 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -752,6 +752,49 @@ def test_completed_event_restores_usage_hidden_by_stream_options_none(): assert completed.response.usage.output_tokens == 5 +def _empty_choices_chunk(usage: Usage | None = None) -> ModelResponseStream: + return ModelResponseStream(id=CHAT_COMPLETION_ID, model="claude-haiku-4-5", choices=[], usage=usage) + + +@pytest.mark.asyncio +async def test_leading_empty_choices_chunk_does_not_kill_the_stream(): + """ + Azure leads some streams with a `prompt_filter_results` chunk whose `choices` is empty. + The bridge used to index `choices[0]` on it and die before the first token. + """ + iterator = _build_iterator([_empty_choices_chunk(), _chunk("Hello"), _chunk("!", finish_reason="stop")]) + + events = [event async for event in iterator] + + event_types = [getattr(event, "type", None) for event in events] + assert event_types.count(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) == 1 + assert "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" + assert event_types[-1] == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + + +@pytest.mark.asyncio +async def test_trailing_empty_choices_usage_chunk_reaches_response_completed(): + """ + With `stream_options.include_usage` (which the bridge always sets) the last upstream chunk + carries only usage and an empty `choices`. It must not crash the stream, and its usage must + still land on `response.completed`. + """ + usage: Final = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + iterator = _build_iterator([_chunk("Hello"), _chunk("", finish_reason="stop"), _empty_choices_chunk(usage)]) + + events = [event async for event in iterator] + + completed = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert completed.response.usage.input_tokens == 10 + assert completed.response.usage.output_tokens == 5 + + +def test_is_reasoning_end_ignores_empty_choices_chunk(): + assert _build_iterator([])._is_reasoning_end(_empty_choices_chunk()) is False + + def test_object_tool_call_arguments_stream_as_valid_json(): """A provider that sends decoded object arguments must still stream valid JSON. diff --git a/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py b/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py index 10d1d6fecd1..250b587aaf1 100644 --- a/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py +++ b/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py @@ -3,14 +3,7 @@ from pathlib import Path import pytest -import litellm from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider -from litellm.types.utils import ( - ImageObject, - ImageResponse, - ImageUsage, - ImageUsageInputTokensDetails, -) REPO_ROOT = Path(__file__).parents[2] MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" @@ -21,94 +14,12 @@ GEMINI = "gemini/gemini-3.1-flash-lite-image" VERTEX = "vertex_ai/gemini-3.1-flash-lite-image" ALL_KEYS = (UNPREFIXED, GEMINI, VERTEX) -INPUT_COST = 2.5e-07 -INPUT_COST_BATCHES = 1.25e-07 -OUTPUT_TEXT_COST = 1.5e-06 -OUTPUT_TEXT_COST_BATCHES = 7.5e-07 -OUTPUT_IMAGE_TOKEN_COST = 3e-05 -OUTPUT_COST_PER_1K_IMAGE = 0.0336 -INPUT_COST_PER_IMAGE = 0.00028 -CACHE_READ_COST = 2.5e-08 -MAX_INPUT_TOKENS = 65536 -MAX_OUTPUT_TOKENS = 4096 -TOKENS_PER_1K_IMAGE = 1120 - -SHARED_FIELDS = { - "mode": "image_generation", - "input_cost_per_token": INPUT_COST, - "input_cost_per_token_batches": INPUT_COST_BATCHES, - "input_cost_per_image": INPUT_COST_PER_IMAGE, - "output_cost_per_token": OUTPUT_TEXT_COST, - "output_cost_per_token_batches": OUTPUT_TEXT_COST_BATCHES, - "output_cost_per_image": OUTPUT_COST_PER_1K_IMAGE, - "output_cost_per_image_token": OUTPUT_IMAGE_TOKEN_COST, - "max_input_tokens": MAX_INPUT_TOKENS, - "max_output_tokens": MAX_OUTPUT_TOKENS, - "max_tokens": MAX_OUTPUT_TOKENS, - "supported_endpoints": ["/v1/chat/completions", "/v1/completions", "/v1/batch"], - "supported_output_modalities": ["text", "image"], - "supports_reasoning": False, - "supports_response_schema": False, - "supports_system_messages": True, - "supports_vision": True, -} - -VERTEX_ROUTE_FIELDS = { - "litellm_provider": "vertex_ai-language-models", - "cache_read_input_token_cost": CACHE_READ_COST, - "supported_modalities": ["text", "image", "video"], - "supports_function_calling": False, - "supports_pdf_input": True, - "supports_prompt_caching": True, - "supports_video_input": True, -} - -PER_ROUTE_FIELDS = { - UNPREFIXED: VERTEX_ROUTE_FIELDS, - VERTEX: VERTEX_ROUTE_FIELDS, - GEMINI: { - "litellm_provider": "gemini", - "supported_modalities": ["text", "image"], - "supports_function_calling": True, - "supports_prompt_caching": False, - "rpm": 1000, - "tpm": 4000000, - }, -} - -GROUNDING_FIELDS = ( - "supports_web_search", - "search_context_cost_per_query", - "web_search_billing_unit", -) - def _load(path: Path) -> dict: with open(path, encoding="utf-8") as f: return json.load(f) -@pytest.fixture -def local_model_cost_map(monkeypatch): - original_model_cost = litellm.model_cost - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.get_model_info.cache_clear() - try: - yield - finally: - litellm.model_cost = original_model_cost - litellm.get_model_info.cache_clear() - - -@pytest.mark.parametrize("model", ALL_KEYS) -@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup")) -def test_per_route_capabilities_match_model_cards(model: str, path: Path): - info = _load(path)[model] - for field, value in PER_ROUTE_FIELDS[model].items(): - assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}" - - @pytest.mark.parametrize("model", ALL_KEYS) def test_backup_matches_main(model: str): assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model) @@ -124,18 +35,3 @@ def test_vertex_prefix_routes_to_vertex(): routed_model, provider, _, _ = get_llm_provider(model=VERTEX) assert routed_model == UNPREFIXED assert provider == "vertex_ai" - - -def _one_k_image_response() -> ImageResponse: - return ImageResponse( - data=[ImageObject(b64_json="img1")], - usage=ImageUsage( - input_tokens=50 + TOKENS_PER_1K_IMAGE, - input_tokens_details=ImageUsageInputTokensDetails( - text_tokens=50, - image_tokens=TOKENS_PER_1K_IMAGE, - ), - output_tokens=TOKENS_PER_1K_IMAGE, - total_tokens=50 + TOKENS_PER_1K_IMAGE + TOKENS_PER_1K_IMAGE, - ), - ) diff --git a/tests/test_litellm/vector_stores/test_main.py b/tests/test_litellm/vector_stores/test_main.py index e3575c33b17..1c968126c42 100644 --- a/tests/test_litellm/vector_stores/test_main.py +++ b/tests/test_litellm/vector_stores/test_main.py @@ -7,6 +7,7 @@ executor, and it must never leak into litellm_params/kwargs where logging would model_dump() it (the #19550 serialization trap). """ +import json from unittest.mock import MagicMock, patch import pytest @@ -15,6 +16,7 @@ import litellm.vector_stores.main as vector_stores_main from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, ) +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.vector_stores.main import search MOCK_SEARCH_RESPONSE = { @@ -89,3 +91,26 @@ def test_search_router_not_in_litellm_params(): litellm_params = mock_handler.call_args.kwargs["litellm_params"] assert "router" not in litellm_params.model_dump(exclude_none=True) assert getattr(litellm_params, "router", None) is None + + +def test_search_forwards_top_level_user_context_to_bedrock_retrieve(): + """Regression (LIT-4415): a top-level userContext, the shape the OpenAI SDK's extra_body + produces on the proxy path, reaches the Bedrock Retrieve request body.""" + client = MagicMock(spec=HTTPHandler) + client.post.return_value = MagicMock(status_code=200, json=MagicMock(return_value={"retrievalResults": []})) + + search( + vector_store_id="kb123", + query="q", + custom_llm_provider="bedrock", + aws_region_name="us-west-2", + aws_access_key_id="test-key-id", + aws_secret_access_key="test-secret-key", + userContext={"userId": "alice@example.com"}, + client=client, + litellm_logging_obj=MagicMock(), + ) + + posted = json.loads(client.post.call_args.kwargs["data"]) + assert posted["userContext"] == {"userId": "alice@example.com"} + assert posted["retrievalQuery"] == {"text": "q"}