Merge remote-tracking branch 'origin/main' into litellm_lit145_timeout_input_cost

.github/workflows/test-unit.yml

@@ -410,6 +410,7 @@ jobs:
               tests/unit/embeddings
               tests/unit/endpoints
               tests/unit/files
+              tests/unit/fine_tuning
               tests/unit/images
               tests/unit/interactions
               tests/unit/messages

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -3,6 +3,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
 """
 
 import asyncio
+import json
 from collections.abc import Mapping, Sequence
 from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
@@ -227,7 +228,7 @@ class CheckBatchCost:
             "so it will no longer be polled"
         )
 
-    async def _claim_job_for_costing(self, job: "ManagedObjectRow") -> bool:
+    async def _claim_job_for_costing(self, job: "ManagedObjectRow", response: "LiteLLMBatch") -> bool:
         """
         Atomically flip batch_processed from false to true, returning whether this pod won
         the row. Every pod and uvicorn worker schedules its own poller against the shared
@@ -241,10 +242,16 @@ class CheckBatchCost:
         """
         if not self._has_batch_processed_column:
             return True
+        from litellm_enterprise.proxy.hooks.managed_files import postgres_storable_json
+
         try:
             claimed: Final = await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": False},
-                data={"batch_processed": True},
+                data={
+                    "batch_processed": True,
+                    "status": response.status if response.status != "completed" else "complete",
+                    "file_object": postgres_storable_json(response),
+                },
             )
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -253,7 +260,9 @@ class CheckBatchCost:
             return False
         return claimed > 0
 
-    async def _release_job_claim(self, job: "ManagedObjectRow") -> None:
+    async def _release_job_claim(
+        self, job: "ManagedObjectRow", original_status: str | None, original_file_object: object
+    ) -> None:
         """Give a claimed row back once billing it failed, so a later poll cycle retries it.
 
         Safe to match on batch_processed=True: while this poller is active the retrieve
@@ -265,7 +274,11 @@ class CheckBatchCost:
         try:
             await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": True},
-                data={"batch_processed": False},
+                data={
+                    "batch_processed": False,
+                    "status": original_status,
+                    "file_object": json.dumps(original_file_object),
+                },
             )
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -564,8 +577,6 @@ class CheckBatchCost:
 
     @staticmethod
     def _get_input_file_id(job: "ManagedObjectRow") -> Optional[str]:
-        import json
-
         from litellm.types.utils import LiteLLMBatch
 
         file_object: Final = job.file_object
@@ -718,8 +729,7 @@ class CheckBatchCost:
                         )
                     except Exception as _e:
                         verbose_proxy_logger.warning(
-                            f"CheckBatchCost: failed to create managed file ID for "
-                            f"{_file_attr}={_raw_file_id!r}: {_e}"
+                            f"CheckBatchCost: failed to create managed file ID for {_file_attr}={_raw_file_id!r}: {_e}"
                         )
 
         # Pass the deployment's router-registered pricing (litellm_params custom
@@ -788,7 +798,9 @@ class CheckBatchCost:
             custom_llm_provider=str(llm_provider) if llm_provider else None,
         )
 
-        if not await self._claim_job_for_costing(job):
+        original_status: Final = job.status
+        original_file_object: Final = job.file_object
+        if not await self._claim_job_for_costing(job, response.model_copy(update={"usage": batch_result.usage})):
             verbose_proxy_logger.info(
                 f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
                 "in this window, so its cost is already being tracked there"
@@ -807,7 +819,7 @@ class CheckBatchCost:
                 batch_completion_cost=batch_result.completion_cost,
             )
         except Exception:
-            await self._release_job_claim(job)
+            await self._release_job_claim(job, original_status, original_file_object)
             raise
 
         # Record batch duration (completed_at - created_at)
@@ -1100,10 +1112,7 @@ class CheckBatchCost:
                 continue
 
             ## RETRIEVE THE BATCH JOB OUTPUT FILE
-            if (
-                response.status in PROVIDER_TERMINAL_BATCH_STATUSES
-                and response.output_file_id is not None
-            ):
+            if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None:
                 try:
                     tracked = await self._track_completed_batch_cost(
                         job=job,
@@ -1134,24 +1143,19 @@ class CheckBatchCost:
                 # Track this job for the final metrics summary
                 processed_models.append(tracked)
 
-                # mark the job as complete
-                try:
-                    update_data: dict = {
-                        "status": response.status if response.status != "completed" else "complete",
-                        "file_object": response.model_dump_json(),
-                    }
-                    if self._has_batch_processed_column:
-                        update_data["batch_processed"] = True
-                    await _managed_object_table(self.prisma_client).update(
-                        where={"id": job.id},
-                        data=update_data,
-                    )
-                except Exception as db_err:
-                    verbose_proxy_logger.error(
-                        f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
-                    )
-                    if not self._has_batch_processed_column:
-                        # nothing claimed the row, so it stays in the poll page and its lines wait for that retry
+                if not self._has_batch_processed_column:
+                    try:
+                        await _managed_object_table(self.prisma_client).update(
+                            where={"id": job.id},
+                            data={
+                                "status": response.status if response.status != "completed" else "complete",
+                                "file_object": response.model_dump_json(),
+                            },
+                        )
+                    except Exception as db_err:
+                        verbose_proxy_logger.error(
+                            f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
+                        )
                         continue
                 self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)
 

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -6,6 +6,7 @@ import base64
 import json
 import time
 from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from itertools import chain
 from types import MappingProxyType
 from typing import (
     TYPE_CHECKING,
@@ -416,7 +417,7 @@ def _storable_json_value(value: JsonValue) -> JsonValue:
     return value
 
 
-def _postgres_storable_json(file_object: BaseModel) -> str:
+def postgres_storable_json(file_object: BaseModel) -> str:
     dumped: Final = file_object.model_dump_json()
     if "\\u0000" not in dumped:
         return dumped
@@ -479,6 +480,13 @@ def _with_managed_background_marker(request: Mapping[str, object]) -> dict[str,
 
 
 _MANAGED_FILES_TARGET: Final = "managed_files"
+_PROVIDER_FILE_ID_KWARGS: Final = MappingProxyType(
+    {
+        CallTypes.acreate_batch: ("input_file_id",),
+        CallTypes.acreate_fine_tuning_job: ("training_file", "validation_file"),
+    }
+)
+_PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDER_FILE_ID_KWARGS.values())))
 _PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
 
 
@@ -650,7 +658,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         )
         # FIX: Update status and file_object on every operation to keep state in sync
         update_columns: Final = {
-            "file_object": _postgres_storable_json(stored_object),
+            "file_object": postgres_storable_json(stored_object),
             "status": file_object.status,
             "updated_by": user_api_key_dict.user_id,
         }
@@ -666,7 +674,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             data={
                 "create": {
                     "unified_object_id": unified_object_id,
-                    "file_object": _postgres_storable_json(stored_object),
+                    "file_object": postgres_storable_json(stored_object),
                     "model_object_id": model_object_id,
                     "file_purpose": file_purpose,
                     "created_by": resolve_resource_owner_id(user_api_key_dict),
@@ -702,7 +710,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "unified_object_id": response_id,
                     "model_object_id": decrypted_response_id(response_id),
                     "file_purpose": "response",
-                    "file_object": _postgres_storable_json(_without_model_output(response)),
+                    "file_object": postgres_storable_json(_without_model_output(response)),
                     "status": PENDING_RESPONSE_STATUSES[0],
                     "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
@@ -1562,6 +1570,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 if isinstance(file_id, str) and file_id
             )
             await self.check_file_ids_access(fine_tuning_file_ids, user_api_key_dict)
+            if fine_tuning_file_ids:
+                data["model_file_id_mapping"] = await self.get_model_file_id_mapping(
+                    fine_tuning_file_ids, user_api_key_dict.parent_otel_span
+                )
 
         return data
 
@@ -1576,24 +1588,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         if request_kwargs is None:
             return healthy_deployments
 
-        input_file_id = cast(Optional[str], request_kwargs.get("input_file_id"))
-        model_file_id_mapping = cast(
-            Optional[Dict[str, Dict[str, str]]],
-            request_kwargs.get("model_file_id_mapping"),
+        request: Final = cast(Mapping[str, object], request_kwargs)
+        model_file_id_mapping: Final = cast(Mapping[str, Mapping[str, str]], request.get("model_file_id_mapping") or {})
+        requested_file_ids: Final = (request.get(key) for key in _PROVIDER_FILE_ID_KEYS)
+        model_ids_per_file: Final = tuple(
+            frozenset(model_file_id_mapping[file_id])
+            for file_id in requested_file_ids
+            if isinstance(file_id, str) and file_id in model_file_id_mapping
+        )
+        allowed_model_ids: Final[frozenset[str]] = (
+            model_ids_per_file[0].intersection(*model_ids_per_file[1:]) if model_ids_per_file else frozenset()
         )
-        allowed_model_ids = []
-        if input_file_id and model_file_id_mapping:
-            model_id_dict = model_file_id_mapping.get(input_file_id, {})
-            allowed_model_ids = list(model_id_dict.keys())
 
         if len(allowed_model_ids) == 0:
             return healthy_deployments
 
-        return [
+        holders: Final[list[dict[str, object]]] = [
             deployment
             for deployment in healthy_deployments
             if deployment.get("model_info", {}).get("id") in allowed_model_ids
         ]
+        return holders if holders or "input_file_id" in request else healthy_deployments
 
     async def async_pre_call_deployment_hook(
         self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
@@ -1601,25 +1616,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         """
         Allow modifying the request just before it's sent to the deployment.
         """
-        accessor_key: Optional[str] = None
-        if call_type and call_type == CallTypes.acreate_batch:
-            accessor_key = "input_file_id"
-        elif call_type and call_type == CallTypes.acreate_fine_tuning_job:
-            accessor_key = "training_file"
-        else:
+        accessor_keys: Final = _PROVIDER_FILE_ID_KWARGS.get(call_type) if call_type else None
+        if accessor_keys is None:
             return kwargs
 
-        if accessor_key:
+        model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
+        # model_info may be at top-level or nested under litellm_metadata
+        # (batch/file operations use litellm_metadata)
+        model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
+        if model_id is None:
+            model_id = cast(
+                Optional[str],
+                kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
+            )
+        for accessor_key in accessor_keys:
             input_file_id = cast(Optional[str], kwargs.get(accessor_key))
-            model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
-            # model_info may be at top-level or nested under litellm_metadata
-            # (batch/file operations use litellm_metadata)
-            model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
-            if model_id is None:
-                model_id = cast(
-                    Optional[str],
-                    kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
-                )
             mapped_file_id: Optional[str] = None
             if input_file_id and model_file_id_mapping and model_id:
                 mapped_file_id = model_file_id_mapping.get(input_file_id, {}).get(model_id, None)
@@ -1813,7 +1824,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     ),
                 )
 
-    async def get_model_file_id_mapping(self, file_ids: List[str], litellm_parent_otel_span: Span) -> dict:
+    async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict:
         """
         Get model-specific file IDs for a list of proxy file IDs.
         Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.77"
+version = "0.1.78"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.77"
+version = "0.1.78"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

helm/litellm-helm/README.md

@@ -70,11 +70,18 @@ proxyConfigMap:
 proxy_config:
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
+  model_list: []
+```
+
+The chart ships no models, so add them under `proxy_config.model_list` or from the Admin UI with `store_model_in_db: true`
+
+```
+proxy_config:
   model_list:
-    - model_name: gpt-3.5-turbo
+    - model_name: gpt-5.6
       litellm_params:
-        model: gpt-3.5-turbo
-        api_key: eXaMpLeOnLy
+        model: openai/gpt-5.6
+        api_key: os.environ/OPENAI_API_KEY
 ```
 
 #### Example using existing `proxyConfigMap` instead of creating it:

helm/litellm-helm/tests/deployment_tests.yaml

@@ -270,6 +270,32 @@ tests:
       - matchRegex:
           path: data["config.yaml"]
           pattern: '(?m)^\s*health_check_details:\s*false$'
+  - it: should not render any placeholder models when the user sets proxy_config without model_list
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.general_settings.store_model_in_db: true
+    asserts:
+      - notMatchRegex:
+          path: data["config.yaml"]
+          pattern: 'model_name:'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'store_model_in_db: true'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'master_key: os.environ/PROXY_MASTER_KEY'
+  - it: should render a user-supplied model_list as the only models
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.model_list:
+        - model_name: my-model
+          litellm_params:
+            model: openai/my-model
+            api_key: os.environ/OPENAI_API_KEY
+    asserts:
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'model_list:\n- litellm_params:\n    api_key: os.environ/OPENAI_API_KEY\n    model: openai/my-model\n  model_name: my-model\n$'
   - it: should allow overriding liveness, readiness, and startup probes
     template: deployment.yaml
     set:

helm/litellm-helm/values.yaml

@@ -180,17 +180,7 @@ proxyConfigMap:
 #  Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
 #  Reference: https://docs.litellm.ai/docs/proxy/configs
 proxy_config:
-  model_list:
-    # At least one model must exist for the proxy to start.
-    - model_name: gpt-3.5-turbo
-      litellm_params:
-        model: gpt-3.5-turbo
-        api_key: eXaMpLeOnLy
-    - model_name: fake-openai-endpoint
-      litellm_params:
-        model: openai/fake
-        api_key: fake-key
-        api_base: https://exampleopenaiendpoint-production.up.railway.app/
+  model_list: []
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
 

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.109"
+version = "0.4.110"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.109"
+version = "0.4.110"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/Cargo.lock

@@ -4083,6 +4083,7 @@ dependencies = [
 name = "litellm-inference-responses"
 version = "0.1.0"
 dependencies = [
+ "base64 0.22.1",
  "bytes",
  "futures-util",
  "litellm-auth",

litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs

@@ -94,6 +94,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
     "docker_model_runner",
     "ragflow",
     "pinstripes",
+    "openinfer",
     "darkbloom",
     "meta",
     "cognition",

litellm-rust/crates/inference-chat/src/common_utils.rs

@@ -6,9 +6,11 @@ use litellm_llms::{
     bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
     openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG,
 };
+use litellm_llms_types::formats::chat_completions::ChatMessage;
 use serde_json::{Map, Value};
 
 use super::Error;
+use crate::types::{ChatCompletionsCall, ResolvedChatCompletionsRequest};
 
 const HEADER_CONTEXT: &str = "chat completions";
 
@@ -42,3 +44,58 @@ pub(super) fn string_headers(
 ) -> Result<Vec<(String, String)>, Error> {
     shared_string_headers(HEADER_CONTEXT, extra_headers).map_err(Error::from)
 }
+
+pub(super) fn resolve_request(
+    call: ChatCompletionsCall,
+    config: &'static dyn BaseConfig,
+) -> Result<ResolvedChatCompletionsRequest, Error> {
+    let messages = serde_json::from_value::<Vec<ChatMessage>>(call.messages).map_err(|err| {
+        Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
+            "chat completions messages",
+            err,
+        ))
+    })?;
+    if messages.is_empty() {
+        return Err(Error::InvalidRequest(
+            "chat completions requires at least one message".into(),
+        ));
+    }
+    if let Some(reason) = config.unsupported_reason(&messages, &call.optional_params) {
+        return Err(Error::Unsupported(reason.0));
+    }
+    Ok(ResolvedChatCompletionsRequest {
+        messages,
+        optional_params: call.optional_params,
+        api_key: call.api_key,
+        api_base: call.api_base,
+        extra_headers: call.extra_headers,
+        timeout: call.timeout,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::chat_completions_provider;
+    use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+
+    #[rstest]
+    #[case::anthropic(LlmProviders::Anthropic, ("anthropic-version", "2023-06-01"))]
+    #[case::bedrock(LlmProviders::Bedrock, ("Content-Type", "application/json"))]
+    fn resolves_supported_provider_configs(
+        #[case] provider: LlmProviders,
+        #[case] expected_header: (&str, &str),
+    ) {
+        let config = chat_completions_provider(provider)
+            .expect("provider has a chat completions config")
+            .config();
+        assert!(config.default_headers().contains(&expected_header));
+    }
+
+    #[rstest]
+    #[case::openai(LlmProviders::Openai)]
+    fn rejects_providers_without_a_chat_config(#[case] provider: LlmProviders) {
+        assert!(chat_completions_provider(provider).is_none());
+    }
+}

litellm-rust/crates/inference-chat/src/handler.rs

@@ -3,44 +3,66 @@ use std::time::Duration;
 
 use litellm_auth::AuthServices;
 use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
-use litellm_http::{Client, outbound::OutboundRequest, request::truncate_error_body};
+use litellm_http::{
+    Client,
+    outbound::OutboundRequest,
+    request::{truncate_error_body, with_default_headers},
+};
 use litellm_llms::base_llm::{
-    auth::{Authenticated, resolve_auth},
+    auth::{Authenticated, ValidatedEnvironment, resolve_auth},
+    chat::transformation::BaseConfig,
     chat::transformation::ProviderChatResponseData,
 };
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
 use super::Error;
-use crate::{constants::CHAT_COMPLETIONS_TIMEOUT_SECS, types::ProviderChatCompletionsRequest};
+use crate::{
+    common_utils::string_headers, constants::CHAT_COMPLETIONS_TIMEOUT_SECS,
+    types::ResolvedChatCompletionsRequest,
+};
 
+#[expect(clippy::too_many_arguments)] // mirrors BaseLLMHTTPHandler.completion, which takes each stage input separately
 pub(super) async fn execute(
     http: &Client,
     auth: &AuthServices,
-    request: ProviderChatCompletionsRequest,
+    config: &'static dyn BaseConfig,
+    provider: litellm_inference::provider::ResolvedProvider<'_>,
+    call: ResolvedChatCompletionsRequest,
+    secrets: Secrets,
     cache: Option<litellm_cache_response::ScopedCache>,
     cache_options: Option<litellm_cache_response::CachePolicy>,
     interceptors: &impl Interceptors<Error>,
     observers: Option<&ObservationSender>,
 ) -> Result<ChatCompletionsResponse, Error> {
-    let ProviderChatCompletionsRequest {
-        model,
-        custom_llm_provider,
-        config,
-        url,
-        body,
-        optional_params,
-        environment,
-        secrets,
-        timeout,
-        api_key,
-    } = request;
+    let validated = config.validate_environment(
+        string_headers(call.extra_headers)?,
+        call.api_key.as_deref(),
+        provider.model,
+        &call.optional_params,
+        &|key| secrets.get(key),
+    )?;
+    let url = config.get_complete_url(
+        call.api_base.as_deref(),
+        provider.model,
+        &call.optional_params,
+        &|key| secrets.get(key),
+    )?;
+    let body = config
+        .transform_request(provider.model, call.messages, call.optional_params.clone())?
+        .body;
+    let environment = ValidatedEnvironment {
+        headers: with_default_headers(validated.headers, config.default_headers()),
+        auth: validated.auth,
+    };
+    let provider_name = <&'static str>::from(provider.provider);
     let context = RequestContext {
-        model: model.clone(),
-        custom_llm_provider,
-        optional_params: Value::Object(optional_params),
+        model: provider.model.to_owned(),
+        custom_llm_provider: provider_name.to_owned(),
+        optional_params: Value::Object(call.optional_params),
         secret_fields: Vec::new(),
-        api_key,
+        api_key: call.api_key.map(litellm_auth::SecretValue::new),
     };
     let authenticated = resolve_auth(auth, environment, &|key| secrets.get(key)).await?;
     let identity = litellm_host::interceptors::ProviderIdentity {
@@ -76,7 +98,7 @@ pub(super) async fn execute(
                 },
                 wire.url,
                 &wire.body,
-                timeout,
+                call.timeout,
             )?;
 
             let response = litellm_inference::outbound::send(outbound, http)
@@ -121,7 +143,7 @@ pub(super) async fn execute(
                 ))
             })?;
             config
-                .transform_response(&model, ProviderChatResponseData { body })
+                .transform_response(provider.model, ProviderChatResponseData { body })
                 .map_err(Error::from)
                 .map_err(as_response_error)
         },
@@ -134,7 +156,7 @@ pub(super) async fn execute(
 ///
 /// A config reports the same variants on either side of the call: a missing
 /// field or an unsupported block can mean "this request cannot be translated"
-/// during prepare and "this response cannot be normalized" here. Only the
+/// before the request is sent and "this response cannot be normalized" here. Only the
 /// second kind has already been billed, and a host that keeps a reference
 /// implementation must not retry those, so collapse them to one variant that
 /// can only mean the provider was already called.
@@ -178,9 +200,10 @@ mod tests {
 
     use super::*;
     use crate::{
-        prepare::{prepare_provider_request, resolve_request},
-        types::ChatCompletionsRequest,
+        common_utils::{chat_completions_provider, resolve_request},
+        types::{ChatCompletionsCall, ResolvedChatCompletionsRequest},
     };
+    use litellm_inference::provider::resolve_llm_provider;
 
     const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
 
@@ -217,22 +240,33 @@ mod tests {
         }
     }
 
-    fn prepared(api_base: &str) -> ProviderChatCompletionsRequest {
-        prepare_provider_request(
-            resolve_request(ChatCompletionsRequest {
-                model: "anthropic/claude-sonnet-4-5",
+    fn prepared(
+        api_base: &str,
+    ) -> (
+        litellm_inference::provider::ResolvedProvider<'static>,
+        &'static dyn BaseConfig,
+        ResolvedChatCompletionsRequest,
+    ) {
+        let provider =
+            resolve_llm_provider("anthropic/claude-sonnet-4-5", None, "chat completions").unwrap();
+        let config = chat_completions_provider(provider.provider)
+            .expect("provider has a chat completions config")
+            .config();
+        let request = resolve_request(
+            ChatCompletionsCall {
+                model: "anthropic/claude-sonnet-4-5".into(),
                 messages: json!([{"role": "user", "content": "hi"}]),
                 optional_params: json!({"max_tokens": 16}).as_object().unwrap().clone(),
-                api_key: Some("sk-test"),
-                api_base: Some(api_base),
+                api_key: Some("sk-test".into()),
+                api_base: Some(api_base.into()),
                 custom_llm_provider: None,
                 extra_headers: None,
                 timeout: None,
-            })
-            .unwrap(),
-            std::sync::Arc::new(|_: &str| None),
+            },
+            config,
         )
-        .unwrap()
+        .unwrap();
+        (provider, config, request)
     }
 
     #[rstest]
@@ -247,10 +281,14 @@ mod tests {
             .await;
         let interceptors = RecordingHooks::default();
 
+        let (provider, config, request) = prepared(&upstream.uri());
         execute(
             &Client::plain_for_test(),
             &AuthServices::default(),
-            prepared(&upstream.uri()),
+            config,
+            provider,
+            request,
+            std::sync::Arc::new(|_: &str| None),
             None,
             None,
             &interceptors,
@@ -288,10 +326,14 @@ mod tests {
             .await;
         let interceptors = RecordingHooks::default();
 
+        let (provider, config, request) = prepared(&upstream.uri());
         let error = execute(
             &Client::plain_for_test(),
             &AuthServices::default(),
-            prepared(&upstream.uri()),
+            config,
+            provider,
+            request,
+            std::sync::Arc::new(|_: &str| None),
             None,
             None,
             &interceptors,

litellm-rust/crates/inference-chat/src/lib.rs

@@ -5,14 +5,13 @@ pub use litellm_inference::RouteError as Error;
 mod common_utils;
 pub mod constants;
 pub(crate) mod handler;
-mod prepare;
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
-use prepare::{prepare_provider_request, resolve_request};
 
 use litellm_auth::AuthServices;
+use litellm_inference::provider::resolve_llm_provider;
 use litellm_secrets::source::SecretSource;
 use std::sync::Arc;
-use types::ChatCompletionsRequest;
+use types::{ChatCompletionsCall, ChatCompletionsRequest};
 
 #[derive(Clone)]
 pub struct ChatCompletionsRoute {
@@ -55,7 +54,7 @@ impl ChatCompletionsRoute {
         } = options.into();
         litellm_host::lifecycle::observe_unary(
             observers.clone(),
-            self.run_call(
+            self.run(
                 request.into(),
                 cache_options,
                 interceptors,
@@ -65,30 +64,52 @@ impl ChatCompletionsRoute {
         .await
     }
 
+    #[tracing::instrument(name = "litellm.route", skip_all, fields(
+        route = "chat_completions",
+        model = %call.model,
+        provider,
+        resolved_model,
+        stream = false,
+        outcome
+    ))]
     async fn run(
         &self,
-        request: ChatCompletionsRequest<'_>,
+        call: ChatCompletionsCall,
         cache_options: Option<litellm_cache_response::CachePolicy>,
         interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
         observers: Option<&ObservationSender>,
     ) -> Result<ChatCompletionsResponse, Error> {
-        let resolved = resolve_request(request)?;
-        let snapshot = self
-            .secrets
-            .resolve(&resolved.config.secret_names())
-            .await?;
-        let prepared = prepare_provider_request(resolved, snapshot)?;
-        litellm_inference::diagnostic::provider(&prepared.model, &prepared.custom_llm_provider);
-        let execute: futures_util::future::BoxFuture<'_, Result<ChatCompletionsResponse, Error>> =
-            Box::pin(handler::execute(
+        litellm_inference::diagnostic::unary(async {
+            let model = call.model.clone();
+            let custom_llm_provider = call.custom_llm_provider.clone();
+            let provider =
+                resolve_llm_provider(&model, custom_llm_provider.as_deref(), "chat completions")?;
+            let config = common_utils::chat_completions_provider(provider.provider)
+                .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?
+                .config();
+            let call = common_utils::resolve_request(call, config)?;
+            litellm_inference::diagnostic::provider(
+                provider.model,
+                <&'static str>::from(provider.provider),
+            );
+            let secrets = self.secrets.resolve(&config.secret_names()).await?;
+            let execute: futures_util::future::BoxFuture<
+                '_,
+                Result<ChatCompletionsResponse, Error>,
+            > = Box::pin(handler::execute(
                 &self.http,
                 &self.auth,
-                prepared,
+                config,
+                provider,
+                call,
+                secrets,
                 self.cache.clone(),
                 cache_options,
                 interceptors,
                 observers,
             ));
-        execute.await
+            execute.await
+        })
+        .await
     }
 }

litellm-rust/crates/inference-chat/src/prepare.rs

@@ -1,638 +0,0 @@
-use litellm_auth::SecretValue;
-use litellm_core_utils::settings::Lookup;
-use litellm_http::request::with_default_headers;
-use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
-use litellm_llms_types::formats::chat_completions::ChatMessage;
-use litellm_secrets::source::Secrets;
-use serde_json::Value;
-
-use super::{
-    Error,
-    common_utils::{chat_completions_provider, string_headers},
-};
-use crate::types::{
-    ChatCompletionsRequest, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest,
-};
-use litellm_inference::provider::resolve_llm_provider;
-
-pub(super) struct ResolvedProvider {
-    pub(super) model: String,
-    pub(super) custom_llm_provider: String,
-    pub(super) config: &'static dyn BaseConfig,
-}
-
-pub(super) fn resolve_provider_config<'a>(
-    model: &'a str,
-    custom_llm_provider: Option<&'a str>,
-) -> Result<ResolvedProvider, Error> {
-    let provider_info = resolve_llm_provider(model, custom_llm_provider, "chat completions")?;
-    let config = chat_completions_provider(provider_info.provider)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?
-        .config();
-    Ok(ResolvedProvider {
-        model: provider_info.model.to_string(),
-        custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
-        config,
-    })
-}
-
-pub(super) fn parse_messages(messages: Value) -> Result<Vec<ChatMessage>, Error> {
-    serde_json::from_value(messages).map_err(|err| {
-        Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
-            "chat completions messages",
-            err,
-        ))
-    })
-}
-
-pub(super) fn resolve_request(
-    request: ChatCompletionsRequest<'_>,
-) -> Result<ResolvedChatCompletionsRequest<'_>, Error> {
-    let ResolvedProvider {
-        model,
-        custom_llm_provider,
-        config,
-    } = resolve_provider_config(request.model, request.custom_llm_provider)?;
-    let messages = parse_messages(request.messages)?;
-    if messages.is_empty() {
-        return Err(Error::InvalidRequest(
-            "chat completions requires at least one message".into(),
-        ));
-    }
-    if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) {
-        return Err(Error::Unsupported(reason.0));
-    }
-    Ok(ResolvedChatCompletionsRequest {
-        model,
-        custom_llm_provider,
-        config,
-        messages,
-        optional_params: request.optional_params,
-        api_key: request.api_key,
-        api_base: request.api_base,
-        extra_headers: request.extra_headers,
-        timeout: request.timeout,
-    })
-}
-
-fn validate_environment(
-    request: &ResolvedChatCompletionsRequest<'_>,
-    model: &str,
-    config: &dyn BaseConfig,
-    secrets: &dyn Lookup,
-) -> Result<ValidatedEnvironment, Error> {
-    let env_lookup = |key: &str| secrets.get(key);
-    let forwarded = string_headers(request.extra_headers.clone())?;
-    let validated = config.validate_environment(
-        forwarded,
-        request.api_key,
-        model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    Ok(ValidatedEnvironment {
-        headers: with_default_headers(validated.headers, config.default_headers()),
-        auth: validated.auth,
-    })
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) fn prepare_provider_request(
-    request: ResolvedChatCompletionsRequest<'_>,
-    secrets: Secrets,
-) -> Result<ProviderChatCompletionsRequest, Error> {
-    let environment =
-        validate_environment(&request, &request.model, request.config, secrets.as_ref())?;
-    let model = request.model;
-    let config = request.config;
-    let env_lookup = |key: &str| secrets.get(key);
-    let url = config.get_complete_url(
-        request.api_base,
-        &model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    let transformed =
-        config.transform_request(&model, request.messages, request.optional_params.clone())?;
-
-    Ok(ProviderChatCompletionsRequest {
-        model,
-        custom_llm_provider: request.custom_llm_provider,
-        config,
-        url,
-        body: transformed.body,
-        optional_params: request.optional_params,
-        environment,
-        secrets,
-        timeout: request.timeout,
-        api_key: request.api_key.map(|key| SecretValue::new(key.to_string())),
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use litellm_auth::CredentialPlacement;
-    use litellm_llms::base_llm::auth::{AuthScheme, resolve_auth};
-    use serde_json::{Map, Value, json};
-
-    use super::{prepare_provider_request, resolve_request};
-    use crate::{
-        Error,
-        types::{ChatCompletionsRequest, ProviderChatCompletionsRequest},
-    };
-
-    fn prepare_chat_completions_call(
-        request: ChatCompletionsRequest<'_>,
-    ) -> Result<ProviderChatCompletionsRequest, Error> {
-        prepare_provider_request(
-            resolve_request(request)?,
-            std::sync::Arc::new(|_: &str| None),
-        )
-    }
-
-    /// The headers as they go on the wire, credential applied.
-    fn wire_headers(prepared: &ProviderChatCompletionsRequest) -> Vec<(String, String)> {
-        tokio::runtime::Builder::new_current_thread()
-            .build()
-            .unwrap()
-            .block_on(resolve_auth(
-                &litellm_auth::AuthServices::default(),
-                prepared.environment.clone(),
-                &|_| None,
-            ))
-            .unwrap()
-            .headers
-    }
-
-    fn request<'a>(
-        model: &'a str,
-        provider: Option<&'a str>,
-        messages: Value,
-        optional_params: Value,
-    ) -> ChatCompletionsRequest<'a> {
-        ChatCompletionsRequest {
-            model,
-            messages,
-            optional_params: match optional_params {
-                Value::Object(map) => map,
-                other => panic!("params must be an object, got {other}"),
-            },
-            api_key: Some("sk-test"),
-            api_base: None,
-            custom_llm_provider: provider,
-            extra_headers: None,
-            timeout: None,
-        }
-    }
-
-    fn preparation_error(request: ChatCompletionsRequest<'_>) -> Error {
-        prepare_chat_completions_call(request)
-            .err()
-            .expect("request preparation should fail")
-    }
-
-    #[test]
-    fn resolves_the_provider_from_the_model_prefix() {
-        let prepared = prepare_chat_completions_call(request(
-            "anthropic/claude-sonnet-4-5",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"max_tokens": 16}),
-        ))
-        .expect("prepares");
-        assert_eq!(prepared.model, "claude-sonnet-4-5");
-        assert_eq!(prepared.url, "https://api.anthropic.com/v1/messages");
-        assert_eq!(prepared.body["model"], json!("claude-sonnet-4-5"));
-    }
-
-    #[test]
-    fn strips_an_explicit_provider_prefix_from_the_model() {
-        let prepared = prepare_chat_completions_call(request(
-            "anthropic/claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        ))
-        .expect("prepares");
-        assert_eq!(prepared.model, "claude-sonnet-4-5");
-    }
-
-    #[test]
-    fn adds_the_auth_and_default_headers() {
-        let prepared = prepare_chat_completions_call(request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        ))
-        .expect("prepares");
-        assert!(
-            wire_headers(&prepared).contains(&("x-api-key".to_string(), "sk-test".to_string()))
-        );
-        assert!(
-            wire_headers(&prepared)
-                .contains(&("anthropic-version".to_string(), "2023-06-01".to_string()))
-        );
-        assert!(matches!(
-            prepared.environment.auth,
-            AuthScheme::Credential {
-                placement: CredentialPlacement::Header("x-api-key"),
-                ..
-            }
-        ));
-    }
-
-    #[test]
-    fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
-        // Python builds `{**headers, **anthropic_headers}`, so the deployment's key
-        // overwrites a forwarded one. Honouring the caller's would let whoever sends
-        // the request choose the Anthropic principal it bills to.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "X-Api-Key".to_string(),
-            json!("sk-caller"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .collect();
-        assert_eq!(keys.len(), 1, "got {:?}", headers);
-        assert_eq!(keys[0].1, "sk-test");
-    }
-
-    #[test]
-    fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
-        // Anthropic's `validate_environment` pops `x-api-key` and sets `authorization`
-        // for an OAuth token, so re-adding the key here would put the credential into
-        // a header the host removed on purpose.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([
-            (
-                "Authorization".to_string(),
-                json!("Bearer sk-ant-oat01-token"),
-            ),
-            ("X-Api-Key".to_string(), json!("sk-caller")),
-        ]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        assert!(
-            !wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("x-api-key") && value == "sk-test"),
-            "the resolved key must not be applied over an OAuth bearer, got {:?}",
-            wire_headers(&prepared)
-        );
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-ant-oat01-token")
-        );
-    }
-
-    #[test]
-    fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
-        // Only an OAuth bearer replaces the credential. Python sends the deployment's
-        // `x-api-key` alongside any other forwarded `authorization`, so deferring on
-        // the mere presence of that header would drop the deployment's auth.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([
-            ("Authorization".to_string(), json!("Bearer unrelated")),
-            ("X-Api-Key".to_string(), json!("sk-caller")),
-        ]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .collect();
-        assert_eq!(keys.len(), 1, "got {:?}", headers);
-        assert_eq!(keys[0].1, "sk-test");
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer unrelated"),
-            "the unrelated authorization must survive, got {:?}",
-            wire_headers(&prepared)
-        );
-    }
-
-    #[rstest::rstest]
-    fn rejects_empty_messages_before_resolving_credentials() {
-        let call = ChatCompletionsRequest {
-            api_key: None,
-            ..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
-        };
-        assert!(matches!(preparation_error(call), Error::InvalidRequest(_)));
-    }
-
-    #[test]
-    fn rejects_an_unknown_provider() {
-        assert_eq!(
-            preparation_error(request(
-                "openai/gpt-4o",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({}),
-            )),
-            Error::InvalidProvider("openai".to_string())
-        );
-    }
-
-    #[test]
-    fn rejects_a_model_with_no_resolvable_provider() {
-        assert!(matches!(
-            preparation_error(request(
-                "claude-sonnet-4-5",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({}),
-            )),
-            Error::InvalidProvider(_)
-        ));
-    }
-
-    #[test]
-    fn rejects_an_empty_or_malformed_message_list() {
-        assert_eq!(
-            preparation_error(request(
-                "anthropic/claude-sonnet-4-5",
-                None,
-                json!([]),
-                json!({}),
-            )),
-            Error::InvalidRequest(
-                "chat completions requires at least one message"
-                    .to_string()
-                    .into()
-            )
-        );
-        assert!(matches!(
-            preparation_error(request(
-                "anthropic/claude-sonnet-4-5",
-                None,
-                json!("not a list"),
-                json!({}),
-            )),
-            Error::InvalidRequest(_)
-        ));
-    }
-
-    #[test]
-    fn rejects_non_string_extra_headers() {
-        let mut call = request(
-            "anthropic/claude-sonnet-4-5",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))]));
-        assert_eq!(
-            preparation_error(call),
-            Error::Headers(litellm_http::request::HeaderError {
-                context: "chat completions",
-                name: "x-trace".to_string(),
-                actual: "number",
-            })
-        );
-    }
-
-    #[test]
-    fn prepares_a_bedrock_call_without_resolving_credentials() {
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        );
-        call.api_key = None;
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        assert_eq!(
-            prepared.url,
-            "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
-        );
-        assert!(matches!(
-            &prepared.environment.auth,
-            AuthScheme::AwsSigV4 { region, service: "bedrock", .. } if region == "us-east-1"
-        ));
-        // SigV4 signs the serialized body, so prepare must not have added an
-        // Authorization header; the signer does it over the bytes sent.
-        assert!(
-            !prepared
-                .environment
-                .headers
-                .iter()
-                .any(|(name, _)| name.eq_ignore_ascii_case("authorization"))
-        );
-        assert_eq!(prepared.body["inferenceConfig"], json!({"maxTokens": 16}));
-    }
-
-    #[tokio::test]
-    async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
-        // Python signs only the AWS header set and reattaches the rest, so a header
-        // the caller forwarded rides along without joining the canonical request.
-        // Signing it makes Converse 403 on a deployment that works on Python.
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({
-                "maxTokens": 16,
-                "aws_access_key_id": "AKIDEXAMPLE",
-                "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
-            }),
-        );
-        // A key would resolve to a bearer token and never reach the signer.
-        call.api_key = None;
-        call.extra_headers = Some(Map::from_iter([(
-            "x-request-id".to_string(),
-            json!("abc-123"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let authenticated = resolve_auth(
-            &litellm_auth::AuthServices::default(),
-            prepared.environment,
-            &|_| None,
-        )
-        .await
-        .expect("resolves");
-        let signed = crate::handler::outbound_request(
-            authenticated,
-            prepared.url,
-            &prepared.body,
-            prepared.timeout,
-        )
-        .expect("signs");
-
-        let authorization = signed
-            .header("authorization")
-            .expect("carries an authorization header")
-            .to_string();
-        assert!(
-            authorization.starts_with("AWS4-HMAC-SHA256"),
-            "expected a SigV4 signature, got {authorization}"
-        );
-        assert!(
-            !authorization.contains("x-request-id"),
-            "forwarded header reached SignedHeaders: {authorization}"
-        );
-        // It still goes on the wire, it is just not part of the signature.
-        assert!(
-            signed
-                .headers()
-                .iter()
-                .any(|(name, value)| name == "x-request-id" && value == "abc-123"),
-            "forwarded header was dropped instead of reattached"
-        );
-    }
-
-    #[rstest::rstest]
-    #[case::authorization("Authorization")]
-    #[case::amz_date("x-amz-date")]
-    #[case::security_token("x-amz-security-token")]
-    #[case::date("Date")]
-    #[tokio::test]
-    async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
-        let call = ChatCompletionsRequest {
-            api_key: None,
-            extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
-            ..request(
-                "bedrock/us-east-1/anthropic.claude-v2",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({
-                    "maxTokens": 16,
-                    "aws_access_key_id": "AKIDEXAMPLE",
-                    "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
-                }),
-            )
-        };
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let authenticated = resolve_auth(
-            &litellm_auth::AuthServices::default(),
-            prepared.environment,
-            &|_| None,
-        )
-        .await
-        .expect("resolves");
-        let error = crate::handler::outbound_request(
-            authenticated,
-            prepared.url,
-            &prepared.body,
-            prepared.timeout,
-        )
-        .expect_err("conflicting signing headers must fail");
-        assert!(
-            matches!(error, Error::Unsupported(_)),
-            "{forwarded} returned {error:?}"
-        );
-    }
-
-    #[test]
-    fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
-        // `get_request_headers` assigns `headers["Authorization"]` unconditionally
-        // once a bearer token resolves, so the deployment's identity wins on
-        // Python. Keeping the caller's would authorize and bill the call as a
-        // different principal, and only when the deployment carries `rust: true`.
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "Authorization".to_string(),
-            json!("Bearer caller-supplied"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let authorizations: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("authorization"))
-            .map(|(_, value)| value.as_str())
-            .collect();
-        assert_eq!(
-            authorizations,
-            vec!["Bearer sk-test"],
-            "the deployment token must be the only authorization on the wire"
-        );
-    }
-
-    #[test]
-    fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
-        // The opposite precedence, and deliberate: Anthropic's own transform
-        // honours a forwarded OAuth bearer, so the Bedrock fix above must not be
-        // generalized into a rule that the configured key always wins.
-        //
-        // An OAuth bearer is the whole of that exception. This forwarded a plain
-        // `x-api-key` until round 17, which read as the same claim and was not:
-        // Python overwrites a forwarded `x-api-key` with the deployment's.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "authorization".to_string(),
-            json!("Bearer sk-ant-oat01-forwarded"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .map(|(_, value)| value.as_str())
-            .collect();
-        assert!(keys.is_empty(), "got {:?}", headers);
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-ant-oat01-forwarded")
-        );
-    }
-
-    #[test]
-    fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
-        // The configured bearer identity has its own account and quota boundary,
-        // so a request carrying one must not be signed as whatever principal the
-        // host's AWS credentials resolve to.
-        let prepared = prepare_chat_completions_call(request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        ))
-        .expect("prepares");
-        assert!(matches!(
-            &prepared.environment.auth,
-            AuthScheme::Credential { placement: CredentialPlacement::Bearer, secret }
-                if secret.expose() == "sk-test"
-        ));
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-test"),
-            "prepare did not carry the bearer token"
-        );
-    }
-}

litellm-rust/crates/inference-chat/src/route.rs

@@ -1,4 +1,3 @@
-use litellm_host::observation::ObservationSender;
 use std::convert::Infallible;
 
 use litellm_host::{
@@ -7,10 +6,7 @@ use litellm_host::{
 };
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
 
-use super::{
-    ChatCompletionsRoute, Error,
-    types::{ChatCompletionsCall, ChatCompletionsRequest},
-};
+use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall};
 
 pub struct ChatCompletions;
 
@@ -37,44 +33,12 @@ impl ChatCompletionsRoute {
             call,
             observers,
             move |call, _, interceptors, observers| async move {
-                self.run_call(call, cache_options, &interceptors, observers.as_ref())
+                self.run(call, cache_options, &interceptors, observers.as_ref())
                     .await
                     .map(CallOutput::Complete)
             },
         )
     }
-
-    #[tracing::instrument(name = "litellm.route", skip_all, fields(
-        route = "chat_completions",
-        model = %call.model,
-        provider,
-        resolved_model,
-        stream = false,
-        outcome
-    ))]
-    pub(super) async fn run_call(
-        &self,
-        call: ChatCompletionsCall,
-        cache_options: Option<litellm_cache_response::CachePolicy>,
-        interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<ChatCompletionsResponse, Error> {
-        litellm_inference::diagnostic::unary(async {
-            let request = ChatCompletionsRequest {
-                model: &call.model,
-                messages: call.messages,
-                optional_params: call.optional_params,
-                api_key: call.api_key.as_deref(),
-                api_base: call.api_base.as_deref(),
-                custom_llm_provider: call.custom_llm_provider.as_deref(),
-                extra_headers: call.extra_headers,
-                timeout: call.timeout,
-            };
-            self.run(request, cache_options, interceptors, observers)
-                .await
-        })
-        .await
-    }
 }
 
 impl litellm_inference::caching::Cachable for ChatCompletions {

litellm-rust/crates/inference-chat/src/types.rs

@@ -1,8 +1,5 @@
-use litellm_secrets::source::Secrets;
 use std::time::Duration;
 
-use litellm_auth::SecretValue;
-use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
 use litellm_llms_types::formats::chat_completions::ChatMessage;
 use serde_json::{Map, Value};
 
@@ -49,31 +46,11 @@ impl From<ChatCompletionsRequest<'_>> for ChatCompletionsCall {
     }
 }
 
-pub struct ResolvedChatCompletionsRequest<'a> {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseConfig,
+pub struct ResolvedChatCompletionsRequest {
     pub messages: Vec<ChatMessage>,
     pub optional_params: Map<String, Value>,
-    pub api_key: Option<&'a str>,
-    pub api_base: Option<&'a str>,
+    pub api_key: Option<String>,
+    pub api_base: Option<String>,
     pub extra_headers: Option<Map<String, Value>>,
     pub timeout: Option<Duration>,
 }
-
-pub struct ProviderChatCompletionsRequest {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseConfig,
-    pub url: String,
-    pub body: Value,
-    /// The route's parameters before the provider transformation, reported to the host
-    /// beside the wire request.
-    pub optional_params: Map<String, Value>,
-    /// The forwarded and default headers plus how the call authenticates; the credential
-    /// itself is applied when the request is sent.
-    pub environment: ValidatedEnvironment,
-    pub secrets: Secrets,
-    pub timeout: Option<Duration>,
-    pub api_key: Option<SecretValue>,
-}

litellm-rust/crates/inference-chat/tests/request.rs

@@ -0,0 +1,439 @@
+use std::time::Duration;
+
+use litellm_inference_chat::{Error, types::ChatCompletionsRequest};
+use rstest::rstest;
+use serde_json::{Map, Value, json};
+use wiremock::ResponseTemplate;
+
+mod support;
+use support::*;
+
+const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
+
+fn request<'a>(
+    model: &'a str,
+    provider: Option<&'a str>,
+    messages: Value,
+    optional_params: Value,
+) -> ChatCompletionsRequest<'a> {
+    let Value::Object(optional_params) = optional_params else {
+        panic!("params must be an object");
+    };
+    ChatCompletionsRequest {
+        model,
+        messages,
+        optional_params,
+        api_key: Some("sk-test"),
+        api_base: None,
+        custom_llm_provider: provider,
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(10)),
+    }
+}
+
+async fn complete(
+    request: ChatCompletionsRequest<'_>,
+) -> Result<litellm_llms_types::formats::chat_completions::ChatCompletionsResponse, Error> {
+    chat_completions_route().execute(request, &(), None).await
+}
+
+fn object(value: Value) -> Map<String, Value> {
+    let Value::Object(map) = value else {
+        panic!("expected an object");
+    };
+    map
+}
+
+fn anthropic_response() -> ResponseTemplate {
+    ResponseTemplate::new(200).set_body_raw(ANTHROPIC_MESSAGE, "application/json")
+}
+
+fn bedrock_response() -> ResponseTemplate {
+    json_response(json!({
+        "output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}},
+        "stopReason": "end_turn",
+        "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}
+    }))
+}
+
+#[rstest]
+#[tokio::test]
+async fn resolves_the_provider_from_the_model_prefix() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "anthropic/claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"max_tokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.url.path(), "/v1/messages");
+    assert_eq!(sent.json()["model"], "claude-sonnet-4-5");
+}
+
+#[rstest]
+#[tokio::test]
+async fn strips_an_explicit_provider_prefix_from_the_model() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "anthropic/claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.json()["model"],
+        "claude-sonnet-4-5"
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn adds_the_auth_and_default_headers() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
+    assert_eq!(sent.header("anthropic-version"), Some("2023-06-01"));
+}
+
+#[rstest]
+#[tokio::test]
+async fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({"X-Api-Key": "sk-caller"}))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("x-api-key"),
+        ["sk-test"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({
+            "Authorization": "Bearer sk-ant-oat01-token",
+            "X-Api-Key": "sk-caller"
+        }))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-caller"]);
+    assert_eq!(
+        sent.header_values("authorization"),
+        ["Bearer sk-ant-oat01-token"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({
+            "Authorization": "Bearer unrelated",
+            "X-Api-Key": "sk-caller"
+        }))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
+    assert_eq!(sent.header_values("authorization"), ["Bearer unrelated"]);
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_empty_messages_before_resolving_credentials() {
+    let error = complete(ChatCompletionsRequest {
+        api_key: None,
+        ..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
+    })
+    .await
+    .expect_err("empty messages are rejected");
+    assert!(matches!(error, Error::InvalidRequest(_)));
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_an_unknown_provider() {
+    assert_eq!(
+        complete(request(
+            "openai/gpt-4o",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        ))
+        .await
+        .unwrap_err(),
+        Error::InvalidProvider("openai".into())
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_a_model_with_no_resolvable_provider() {
+    assert!(matches!(
+        complete(request(
+            "claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        ))
+        .await,
+        Err(Error::InvalidProvider(_))
+    ));
+}
+
+#[rstest]
+#[case::empty(json!([]), Some("chat completions requires at least one message"))]
+#[case::malformed(json!("not a list"), None)]
+#[tokio::test]
+async fn rejects_an_empty_or_malformed_message_list(
+    #[case] messages: Value,
+    #[case] expected: Option<&str>,
+) {
+    let error = complete(request(
+        "anthropic/claude-sonnet-4-5",
+        None,
+        messages,
+        json!({}),
+    ))
+    .await
+    .unwrap_err();
+    if let Some(expected) = expected {
+        assert_eq!(error, Error::InvalidRequest(expected.to_string().into()));
+    } else {
+        assert!(matches!(error, Error::InvalidRequest(_)));
+    }
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_non_string_extra_headers() {
+    assert_eq!(
+        complete(ChatCompletionsRequest {
+            extra_headers: Some(Map::from_iter([("x-trace".to_string(), json!(7))])),
+            ..request(
+                "anthropic/claude-sonnet-4-5",
+                None,
+                json!([{"role": "user", "content": "hi"}]),
+                json!({}),
+            )
+        })
+        .await
+        .unwrap_err(),
+        Error::Headers(litellm_http::request::HeaderError {
+            context: "chat completions",
+            name: "x-trace".to_string(),
+            actual: "number",
+        })
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    let mut call = request(
+        "bedrock/us-east-1/anthropic.claude-v2",
+        None,
+        json!([{"role": "user", "content": "hi"}]),
+        json!({
+            "maxTokens": 16,
+            "aws_access_key_id": "AKIDEXAMPLE",
+            "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
+        }),
+    );
+    call.api_key = None;
+    call.api_base = Some(&base);
+    call.extra_headers = Some(Map::from_iter([(
+        "x-request-id".to_string(),
+        json!("abc-123"),
+    )]));
+    complete(call).await.expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    let authorization = sent.header("authorization").expect("signed request");
+    assert!(authorization.starts_with("AWS4-HMAC-SHA256"));
+    assert!(!authorization.contains("x-request-id"));
+    assert_eq!(sent.header("x-request-id"), Some("abc-123"));
+    assert_eq!(sent.url.path(), "/model/anthropic.claude-v2/converse");
+    let body: Value = serde_json::from_slice(&sent.body).unwrap();
+    assert_eq!(body["inferenceConfig"], json!({"maxTokens": 16}));
+    assert_eq!(
+        body["messages"],
+        json!([{"role": "user", "content": [{"text": "hi"}]}])
+    );
+}
+
+#[rstest]
+#[case::authorization("Authorization")]
+#[case::amz_date("x-amz-date")]
+#[case::security_token("x-amz-security-token")]
+#[case::date("Date")]
+#[tokio::test]
+async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
+    let upstream = wiremock::MockServer::start().await;
+    let base = upstream.uri();
+    let call = ChatCompletionsRequest {
+        api_key: None,
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({
+                "maxTokens": 16,
+                "aws_access_key_id": "AKIDEXAMPLE",
+                "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
+            }),
+        )
+    };
+    let error = complete(call)
+        .await
+        .expect_err("conflicting signing headers fail");
+    assert!(matches!(error, Error::Unsupported(_)), "{error:?}");
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(
+            "Authorization".to_string(),
+            json!("Bearer caller-supplied"),
+        )])),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"maxTokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("authorization"),
+        ["Bearer sk-test"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(
+            "authorization".to_string(),
+            json!("Bearer sk-ant-oat01-forwarded"),
+        )])),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert!(sent.header_values("x-api-key").is_empty());
+    assert_eq!(
+        sent.header_values("authorization"),
+        ["Bearer sk-ant-oat01-forwarded"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"maxTokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("authorization"),
+        ["Bearer sk-test"]
+    );
+}

litellm-rust/crates/inference-messages/AGENTS.md

@@ -7,8 +7,8 @@
   - delegate authentication policy, beta selection, payload rewriting, and response interpretation to those adapters
   - keep provider policy out of request preparation and transport handlers
   - calling a concrete provider helper for every provider is still a policy dependency
-- Route types such as `MessagesCall`, prepared requests, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
+- Route types such as `MessagesCall`, provider calls, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
 - The route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks
-- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `src/lib.rs` explicitly sequences cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
+- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `handler::execute` explicitly sequences the provider call build in Python's handler order, cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
 - Preserve the order of validation, normalization, caller-requested parameter removal, and provider transformation when that order affects observable behavior
 - Test provider dispatch, auth precedence, header handling, transformations, and responses through behavior, not source structure

litellm-rust/crates/inference-messages/src/common_utils.rs

@@ -82,17 +82,17 @@ mod tests {
     use litellm_core_utils::get_llm_provider_logic::LlmProviders;
 
     #[rstest]
-    #[case::tencent("tencent")]
-    #[case::anthropic("anthropic")]
-    #[case::azure_ai("azure_ai")]
-    #[case::bedrock("bedrock")]
-    #[case::deepseek("deepseek")]
-    #[case::edenai("edenai")]
-    #[case::minimax("minimax")]
-    #[case::vertex_ai("vertex_ai")]
-    fn provider_keeps_its_python_name(#[case] name: &str) {
-        let provider =
-            messages_provider(name.parse::<LlmProviders>().unwrap(), "claude-sonnet-4-5").unwrap();
+    #[case::tencent(LlmProviders::Tencent, "tencent")]
+    #[case::anthropic(LlmProviders::Anthropic, "anthropic")]
+    #[case::azure_ai(LlmProviders::AzureAi, "azure_ai")]
+    #[case::bedrock(LlmProviders::Bedrock, "bedrock")]
+    #[case::deepseek(LlmProviders::Deepseek, "deepseek")]
+    #[case::edenai(LlmProviders::Edenai, "edenai")]
+    #[case::minimax(LlmProviders::Minimax, "minimax")]
+    #[case::vertex_ai(LlmProviders::VertexAi, "vertex_ai")]
+    #[case::openrouter(LlmProviders::Openrouter, "openrouter")]
+    fn provider_keeps_its_python_name(#[case] provider: LlmProviders, #[case] name: &str) {
+        let provider = messages_provider(provider, "claude-sonnet-4-5").unwrap();
         assert_eq!(provider.as_str(), name);
     }
 

litellm-rust/crates/inference-messages/src/handler.rs

@@ -1,148 +1,271 @@
 use std::time::Duration;
 
 use bytes::Bytes;
-use futures_util::{StreamExt, TryStreamExt, stream::BoxStream};
-use litellm_host::interceptors::{Interceptors, ProviderIdentity, RequestContext, WireRequest};
-use litellm_http::transport::Error as TransportError;
+use futures_util::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream};
+use litellm_auth::{AuthServices, SecretValue};
+use litellm_core_utils::{
+    dot_notation_indexing::delete_nested_value,
+    get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
+};
+use litellm_host::interceptors::{
+    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
+};
+use litellm_http::{request::with_default_headers, transport::Error as TransportError};
 use litellm_llms::base_llm::{
-    auth::{Authenticated, resolve_auth},
+    auth::{Authenticated, ValidatedEnvironment, resolve_auth},
     messages::{
+        context::MessagesTransformContext,
         streaming::{self, ByteStream, StreamDecoder},
         transformation::BaseMessagesConfig,
     },
 };
-use litellm_llms_types::formats::messages::MessagesResponse;
+use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_secrets::source::Secrets;
 use litellm_tracing::ByteChunk;
 use serde_json::Value;
 
 use super::{
-    Error, MessagesCallResponse, MessagesRoute, common_utils::truncate_error_body,
-    prepare::ProviderMessagesRequest,
+    Error, MessagesCall, MessagesCallResponse, MessagesRoute,
+    common_utils::{MessagesProvider, string_headers, truncate_error_body},
+    constants::MESSAGES_TIMEOUT_SECS,
+    route::Messages,
+    types::invalid_request,
 };
-use crate::constants::MESSAGES_TIMEOUT_SECS;
-use litellm_inference::{context::CallContext, outbound::outbound_request};
+use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request};
 
-pub(super) struct ProviderCall {
-    pub identity: ProviderIdentity,
-    pub wire: WireRequest,
-    pub(super) provider: super::common_utils::MessagesProvider,
+struct ProviderCall {
+    identity: ProviderIdentity,
+    wire: WireRequest,
+    provider: MessagesProvider,
     signer: Option<litellm_auth_aws::SigV4Signer>,
     timeout: Option<Duration>,
     stream: bool,
 }
 
 impl ProviderCall {
-    pub fn cacheable(&self) -> bool {
+    fn cacheable(&self) -> bool {
         self.signer.is_none()
     }
 }
 
-impl MessagesRoute {
-    pub(super) async fn prepare_outbound(
-        &self,
-        request: ProviderMessagesRequest,
-        context: &CallContext<'_, impl Interceptors<Error>>,
-    ) -> Result<ProviderCall, Error> {
-        let ProviderMessagesRequest {
-            provider,
-            url,
-            body,
-            environment,
-            timeout,
-            api_key,
-        } = request;
-        let request_context = RequestContext {
-            model: body.model.clone(),
-            custom_llm_provider: provider.as_str().to_string(),
-            optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
-            secret_fields: Vec::new(),
-            api_key,
-        };
-        let authenticated =
-            resolve_auth(&self.auth, environment, &|key| std::env::var(key).ok()).await?;
-        let identity = ProviderIdentity {
-            model: request_context.model.clone(),
-            provider: request_context.custom_llm_provider.clone(),
-        };
-        let wire = context
-            .interceptors
-            .before_provider_request(
-                WireRequest {
-                    url,
-                    headers: authenticated.headers,
-                    body: provider
-                        .config()
-                        .wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
-                },
-                request_context,
-            )
-            .await?;
-        let stream = match wire.body.get("stream") {
-            None | Some(Value::Null) => false,
-            Some(Value::Bool(stream)) => *stream,
-            Some(value) => {
-                return Err(Error::InvalidRequest(
-                    litellm_llms::ErrorDetail::InvalidValue {
-                        field: "stream",
-                        expected: "a boolean",
-                        actual: value.clone(),
-                    },
-                ));
-            }
-        };
-        Ok(ProviderCall {
-            identity,
-            wire,
-            provider,
-            signer: authenticated.signer,
-            timeout,
-            stream,
+pub(super) async fn execute(
+    route: &MessagesRoute,
+    provider: MessagesProvider,
+    model: String,
+    call: MessagesCall,
+    secrets: Secrets,
+    context: &CallContext<'_, impl Interceptors<Error>>,
+) -> Result<MessagesCallResponse, Error> {
+    let request = provider_call(
+        &route.auth,
+        provider,
+        model,
+        call,
+        secrets.as_ref(),
+        context,
+    )
+    .boxed()
+    .await?;
+    let cache = CallCache::<Messages>::from_wire(
+        route.cache.as_ref().filter(|_| request.cacheable()),
+        context.cache,
+        &request.identity,
+        &request.wire,
+    );
+    let identity = request.identity.clone();
+    let (output, source) = match cache.lookup().await {
+        Some(hit) => hit,
+        None => (
+            call_provider(&route.http, request, context).await?,
+            ResultSource::Provider,
+        ),
+    };
+    let reported = match &output {
+        MessagesCallResponse::Complete(response) => provider.config().reported_cost(response),
+        MessagesCallResponse::Stream { .. } => None,
+    };
+    context
+        .result_ready(ExecutionFacts {
+            provider: identity,
+            cost: Cost::settle(&source, reported),
+            source: source.clone(),
         })
-    }
+        .await?;
+    Ok(cache.finish(output, &source).await)
+}
 
-    pub(super) async fn call_provider(
-        &self,
-        request: ProviderCall,
-        context: &CallContext<'_, impl Interceptors<Error>>,
-    ) -> Result<MessagesCallResponse, Error> {
-        let ProviderCall {
-            identity,
-            wire,
-            provider,
-            signer,
-            timeout,
-            stream,
-        } = request;
-        let provider_name = provider.as_str();
-        log_request_body(provider_name, stream, &wire.body);
-        let response = send(
-            &self.http,
-            Authenticated {
-                headers: wire.headers,
-                signer,
+async fn provider_call(
+    auth: &AuthServices,
+    provider: MessagesProvider,
+    model: String,
+    call: MessagesCall,
+    secrets: &(dyn Lookup + Send + Sync),
+    context: &CallContext<'_, impl Interceptors<Error>>,
+) -> Result<ProviderCall, Error> {
+    let MessagesCall {
+        body,
+        api_key,
+        api_base,
+        litellm_params,
+        extra_headers,
+        provider_specific_header,
+        timeout,
+        shaping,
+        ..
+    } = call;
+    let connection_params = litellm_router_types::LitellmParams {
+        api_base: api_base.clone().or(litellm_params.api_base.clone()),
+        ..litellm_params
+    };
+    let config = provider.config();
+    let env_lookup = |key: &str| secrets.get(key);
+
+    let sanitized = config.shape_request(
+        MessagesRequest { model, ..body },
+        shaping.settings.reasoning_auto_summary,
+    )?;
+    let trimmed =
+        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
+
+    let scoped =
+        get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
+    let forwarded = string_headers(Some(
+        extra_headers.into_iter().flatten().chain(scoped).collect(),
+    ))?;
+    let validated = config.validate_environment(
+        forwarded,
+        api_key.as_deref(),
+        &trimmed.model,
+        &trimmed.params,
+        &connection_params,
+        &env_lookup,
+    )?;
+
+    let body = config.transform_anthropic_messages_request(
+        trimmed,
+        &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
+    )?;
+    let environment = ValidatedEnvironment {
+        headers: config.request_headers(
+            with_default_headers(validated.headers, config.default_headers()),
+            &body,
+        ),
+        auth: validated.auth,
+    };
+
+    let url = config.get_complete_url(
+        api_base.as_deref(),
+        &body.model,
+        &connection_params,
+        body.params.stream == Some(true),
+        &env_lookup,
+    )?;
+
+    let request_context = RequestContext {
+        model: body.model.clone(),
+        custom_llm_provider: provider.as_str().to_string(),
+        optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
+        secret_fields: Vec::new(),
+        api_key: api_key.map(SecretValue::new),
+    };
+    let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?;
+    let identity = ProviderIdentity {
+        model: request_context.model.clone(),
+        provider: request_context.custom_llm_provider.clone(),
+    };
+    let wire = context
+        .interceptors
+        .before_provider_request(
+            WireRequest {
+                url,
+                headers: authenticated.headers,
+                body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
             },
-            &wire.url,
-            &wire.body,
-            timeout,
+            request_context,
         )
         .await?;
-        if !response.status().is_success() {
-            return Err(provider_error(response).await);
-        }
-        let config = provider.config();
-        if stream {
-            return Ok(streaming_response(
-                response,
-                config.stream_decoder(),
-                provider_name,
+    let stream = match wire.body.get("stream") {
+        None | Some(Value::Null) => false,
+        Some(Value::Bool(stream)) => *stream,
+        Some(value) => {
+            return Err(Error::InvalidRequest(
+                litellm_llms::ErrorDetail::InvalidValue {
+                    field: "stream",
+                    expected: "a boolean",
+                    actual: value.clone(),
+                },
             ));
         }
-        let text = response.text().await.map_err(network)?;
-        log_response_body(&text);
-        context.response_received(&text).await?;
-        decode_response(config, &identity.model, &text)
-            .map(|message| MessagesCallResponse::Complete(Box::new(message)))
+    };
+    Ok(ProviderCall {
+        identity,
+        wire,
+        provider,
+        signer: authenticated.signer,
+        timeout: timeout.or_else(|| shaping.settings.request_timeout()),
+        stream,
+    })
+}
+
+fn without_additional_drop_params(
+    request: MessagesRequest,
+    paths: &[String],
+) -> Result<MessagesRequest, Error> {
+    if paths.is_empty() {
+        return Ok(request);
     }
+    let params = serde_json::to_value(request.params).map_err(invalid_request)?;
+    let trimmed = paths
+        .iter()
+        .fold(params, |params, path| delete_nested_value(params, path));
+    Ok(MessagesRequest {
+        params: serde_json::from_value(trimmed).map_err(invalid_request)?,
+        ..request
+    })
+}
+
+async fn call_provider(
+    http: &litellm_http::Client,
+    request: ProviderCall,
+    context: &CallContext<'_, impl Interceptors<Error>>,
+) -> Result<MessagesCallResponse, Error> {
+    let ProviderCall {
+        identity,
+        wire,
+        provider,
+        signer,
+        timeout,
+        stream,
+    } = request;
+    let provider_name = provider.as_str();
+    log_request_body(provider_name, stream, &wire.body);
+    let response = send(
+        http,
+        Authenticated {
+            headers: wire.headers,
+            signer,
+        },
+        &wire.url,
+        &wire.body,
+        timeout,
+    )
+    .await?;
+    if !response.status().is_success() {
+        return Err(provider_error(response).await);
+    }
+    let config = provider.config();
+    if stream {
+        return Ok(streaming_response(
+            response,
+            config.stream_decoder(),
+            provider_name,
+        ));
+    }
+    let text = response.text().await.map_err(network)?;
+    log_response_body(&text);
+    context.response_received(&text).await?;
+    decode_response(config, &identity.model, &text)
+        .map(|message| MessagesCallResponse::Complete(Box::new(message)))
 }
 
 fn serialize_failure(err: serde_json::Error) -> Error {
@@ -271,10 +394,482 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) {
 
 #[cfg(test)]
 mod tests {
-    use rstest::rstest;
+    use litellm_llms_types::headers::ProviderSpecificHeaders;
+    use rstest::{fixture, rstest};
+    use serde_json::{Map, Value, json};
     use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
 
     use super::*;
+    use crate::{MessagesSettings, MessagesShaping, common_utils::messages_provider};
+    use litellm_inference::provider::resolve_llm_provider;
+
+    #[fixture]
+    fn shaping() -> MessagesShaping {
+        MessagesShaping::default()
+    }
+
+    fn body(value: Value) -> MessagesRequest {
+        serde_json::from_value(value).unwrap()
+    }
+
+    async fn provider_call_for(call: MessagesCall) -> Result<ProviderCall, Error> {
+        provider_call_with_secrets(call, &|_: &str| None).await
+    }
+
+    async fn provider_call_with_secrets(
+        call: MessagesCall,
+        secrets: &(dyn Lookup + Send + Sync),
+    ) -> Result<ProviderCall, Error> {
+        let provider = resolve_llm_provider(
+            &call.body.model,
+            call.custom_llm_provider.as_deref(),
+            "messages",
+        )?;
+        let config = messages_provider(provider.provider, provider.model)
+            .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?;
+        let model = provider.model.to_owned();
+        let interceptors = ();
+        let context = CallContext::new(&interceptors, litellm_inference::CallOptions::default());
+        provider_call(
+            &AuthServices::default(),
+            config,
+            model,
+            call,
+            secrets,
+            &context,
+        )
+        .await
+    }
+
+    async fn wire_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
+        provider_call_for(MessagesCall {
+            body: body(fields),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://anthropic.test".into()),
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        })
+        .await
+        .map(|call| call.wire.body)
+    }
+
+    #[rstest]
+    #[case::api_key(
+        &[("ANTHROPIC_API_KEY", "sk-secret")],
+        &[("x-api-key", "sk-secret")],
+        "https://api.anthropic.com/v1/messages"
+    )]
+    #[case::auth_token(
+        &[("ANTHROPIC_AUTH_TOKEN", "token")],
+        &[("authorization", "Bearer token")],
+        "https://api.anthropic.com/v1/messages"
+    )]
+    #[case::api_base(
+        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
+        &[("x-api-key", "sk-secret")],
+        "https://gateway.test/v1/messages"
+    )]
+    #[case::sdk_base_url(
+        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
+        &[("x-api-key", "sk-secret")],
+        "https://sdk.test/v1/messages"
+    )]
+    #[tokio::test]
+    async fn credentials_and_base_come_from_the_resolved_secrets(
+        shaping: MessagesShaping,
+        #[case] secrets: &[(&str, &str)],
+        #[case] expected_auth: &[(&str, &str)],
+        #[case] expected_url: &str,
+    ) {
+        let lookup = |name: &str| {
+            secrets
+                .iter()
+                .find(|(key, _)| *key == name)
+                .map(|(_, value)| value.to_string())
+        };
+        let call = provider_call_with_secrets(
+            MessagesCall {
+                body: body(
+                    json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
+                ),
+                api_key: None,
+                api_base: None,
+                custom_llm_provider: Some("anthropic".into()),
+                litellm_params: Default::default(),
+                extra_headers: None,
+                provider_specific_header: None,
+                timeout: None,
+                shaping,
+            },
+            &lookup,
+        )
+        .await
+        .unwrap();
+        let auth: Vec<(&str, &str)> = call
+            .wire
+            .headers
+            .iter()
+            .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
+            .map(|(name, value)| (name.as_str(), value.as_str()))
+            .collect();
+        assert_eq!(
+            (auth.as_slice(), call.wire.url.as_str()),
+            (expected_auth, expected_url)
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn missing_credentials_are_reported_before_the_request_is_transformed(
+        shaping: MessagesShaping,
+    ) {
+        let result = provider_call_for(MessagesCall {
+            body: body(
+                json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}]}),
+            ),
+            api_key: None,
+            api_base: None,
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        })
+        .await;
+        assert!(
+            matches!(
+                result,
+                Err(Error::Auth(litellm_auth::Error::MissingApiKey { .. }))
+            ),
+            "{:?}",
+            result.err()
+        );
+    }
+
+    #[rstest]
+    #[case::nothing_configured(None, None, None)]
+    #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
+    #[case::call_timeout_wins(
+        Some(Duration::from_secs(30)),
+        Some(1800.0),
+        Some(Duration::from_secs(30))
+    )]
+    #[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
+    #[case::unusable_setting_ignored(None, Some(-1.0), None)]
+    #[case::zero_setting_ignored(None, Some(0.0), None)]
+    #[tokio::test]
+    async fn timeout_falls_back_to_the_configured_request_timeout(
+        shaping: MessagesShaping,
+        #[case] call_timeout: Option<Duration>,
+        #[case] request_timeout: Option<f64>,
+        #[case] expected: Option<Duration>,
+    ) {
+        let call = provider_call_for(MessagesCall {
+            body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://anthropic.test".into()),
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: call_timeout,
+            shaping: MessagesShaping {
+                settings: MessagesSettings { request_timeout, ..shaping.settings },
+                ..shaping
+            },
+        })
+        .await
+        .unwrap();
+        assert_eq!(call.timeout, expected);
+    }
+
+    #[rstest]
+    #[case::top_level_and_nested_paths(
+        json!({
+            "max_tokens": 1024,
+            "thinking": {"type": "enabled", "budget_tokens": 2048},
+            "context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
+            "metadata": {"user_id": "u1"},
+            "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
+        }),
+        &["thinking", "context_management", "tools[*].input_examples"],
+        json!({
+            "max_tokens": 1024,
+            "metadata": {"user_id": "u1"},
+            "tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
+        }),
+    )]
+    #[case::no_paths(
+        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[],
+        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
+    )]
+    #[case::model_and_messages_are_never_dropped(
+        json!({"max_tokens": 16}),
+        &["model", "messages", "messages[0].content"],
+        json!({"max_tokens": 16}),
+    )]
+    #[tokio::test]
+    async fn wire_body_drops_configured_paths(
+        shaping: MessagesShaping,
+        #[case] fields: Value,
+        #[case] additional_drop_params: &[&str],
+        #[case] expected_fields: Value,
+    ) {
+        let with_messages = |fields: Value| -> Value {
+            let Value::Object(fields) = fields else {
+                unreachable!()
+            };
+            Value::Object(
+                [
+                    ("model".to_string(), json!("claude-test")),
+                    (
+                        "messages".to_string(),
+                        json!([{"role": "user", "content": "hi"}]),
+                    ),
+                ]
+                .into_iter()
+                .chain(fields)
+                .collect(),
+            )
+        };
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                additional_drop_params: additional_drop_params
+                    .iter()
+                    .map(ToString::to_string)
+                    .collect(),
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert_eq!(
+            wire_body(with_messages(fields), shaping).await,
+            Ok(with_messages(expected_fields))
+        );
+    }
+
+    #[rstest]
+    #[case::model_prefix_picks_the_provider(
+        "azure_ai/claude-test",
+        None,
+        &[("x-priority", "extra"), ("x-scoped", "azure_ai")]
+    )]
+    #[case::explicit_provider(
+        "claude-test",
+        Some("anthropic"),
+        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
+    )]
+    #[case::provider_prefix_on_an_anthropic_model(
+        "anthropic/claude-test",
+        None,
+        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
+    )]
+    #[tokio::test]
+    async fn provider_specific_headers_follow_the_resolved_provider(
+        shaping: MessagesShaping,
+        #[case] model: &str,
+        #[case] custom_llm_provider: Option<&str>,
+        #[case] expected: &[(&str, &str)],
+    ) {
+        let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
+            {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
+            {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
+        ]))
+        .unwrap();
+        let call = provider_call_for(MessagesCall {
+            body: body(
+                json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
+            ),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://resource.services.ai.azure.com".into()),
+            custom_llm_provider: custom_llm_provider.map(Into::into),
+            litellm_params: Default::default(),
+            extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
+            provider_specific_header: Some(configured),
+            timeout: None,
+            shaping,
+        })
+        .await
+        .unwrap();
+        let caller_headers: Vec<(&str, &str)> = call
+            .wire
+            .headers
+            .iter()
+            .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
+            .map(|(name, value)| (name.as_str(), value.as_str()))
+            .collect();
+        assert_eq!(caller_headers, expected);
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn wire_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "anthropic/claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16
+                }),
+                shaping,
+            )
+            .await,
+            Ok(json!({
+                "model": "claude-test",
+                "messages": [{"role": "user", "content": "hi"}],
+                "max_tokens": 16
+            }))
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                reasoning_auto_summary: true,
+                additional_drop_params: vec!["thinking.display".to_string()],
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 4096,
+                    "thinking": {"type": "enabled", "budget_tokens": 2048}
+                }),
+                shaping,
+            )
+            .await,
+            Ok(json!({
+                "model": "claude-test",
+                "messages": [{"role": "user", "content": "hi"}],
+                "max_tokens": 4096,
+                "thinking": {"type": "enabled", "budget_tokens": 2048}
+            }))
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(
+        shaping: MessagesShaping,
+    ) {
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                additional_drop_params: vec!["metadata.user_id".to_string()],
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert!(matches!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16,
+                    "metadata": {"user_id": 123}
+                }),
+                shaping,
+            )
+            .await,
+            Err(Error::InvalidRequest(_))
+        ));
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn wire_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16,
+                    "metadata": {"user_id": 123}
+                }),
+                shaping,
+            )
+            .await,
+            Err(Error::InvalidRequest(
+                litellm_llms::ErrorDetail::InvalidValue {
+                    field: "metadata.user_id",
+                    expected: "a string",
+                    actual: json!(123),
+                }
+            ))
+        );
+    }
+
+    #[rstest]
+    #[case::mantle(
+        "bedrock_mantle/claude-test",
+        "https://bedrock-mantle.us-west-2.api.aws",
+        "/anthropic/v1/messages",
+        "bedrock"
+    )]
+    #[case::platform(
+        "bedrock/claude_platform/claude-test",
+        "https://aws-external-anthropic.us-west-2.api.aws",
+        "/v1/messages",
+        "aws-external-anthropic"
+    )]
+    #[tokio::test]
+    async fn explicit_endpoint_region_is_used_by_the_signer(
+        shaping: MessagesShaping,
+        #[case] model: &str,
+        #[case] base: &str,
+        #[case] path: &str,
+        #[case] service: &str,
+    ) {
+        let call = provider_call_with_secrets(MessagesCall {
+            body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
+            api_key: None,
+            api_base: Some(base.into()),
+            custom_llm_provider: None,
+            litellm_params: crate::litellm_params(Map::from_iter([
+                ("model".into(), json!(model)),
+                ("aws_access_key_id".into(), json!("AKIDEXAMPLE")),
+                ("aws_secret_access_key".into(), json!("secret")),
+            ]))
+            .unwrap(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        }, &|name: &str| match name {
+            "AWS_REGION_NAME" => Some("us-east-1".into()),
+            "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
+            _ => None,
+        })
+        .await
+        .unwrap();
+        assert_eq!(call.wire.url, format!("{base}{path}"));
+        let signed = outbound_request(
+            Authenticated {
+                headers: call.wire.headers,
+                signer: call.signer,
+            },
+            call.wire.url,
+            &call.wire.body,
+            None,
+        )
+        .unwrap();
+        let authorization = signed.header("authorization").unwrap_or_default();
+        assert!(
+            authorization.contains(&format!("/us-west-2/{service}/aws4_request")),
+            "{authorization}"
+        );
+    }
 
     #[rstest]
     #[case::event(

litellm-rust/crates/inference-messages/src/lib.rs

@@ -1,18 +1,19 @@
 mod common_utils;
 mod constants;
 mod handler;
-mod prepare;
 pub mod route;
 mod types;
 
 use futures_util::FutureExt;
 use litellm_auth::AuthServices;
-use litellm_host::interceptors::{Cost, ExecutionFacts, Interceptors, ResultSource};
-
-use litellm_inference::{caching::CallCache, context::CallContext};
+use litellm_host::interceptors::Interceptors;
+use litellm_inference::context::CallContext;
+use litellm_inference::provider::resolve_llm_provider;
 use litellm_secrets::source::SecretSource;
 use std::sync::Arc;
 
+use crate::common_utils::messages_provider;
+
 pub use litellm_inference::RouteError as Error;
 pub use types::{
     MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
@@ -74,39 +75,23 @@ impl MessagesRoute {
         context: CallContext<'_, impl Interceptors<Error>>,
     ) -> Result<MessagesCallResponse, Error> {
         litellm_inference::diagnostic::call(async {
-            let prepared = prepare::prepare(call, self.secrets.as_ref()).await?;
+            let provider = resolve_llm_provider(
+                &call.body.model,
+                call.custom_llm_provider.as_deref(),
+                "messages",
+            )?;
+            let config = messages_provider(provider.provider, provider.model).ok_or_else(|| {
+                Error::InvalidProvider(<&str>::from(provider.provider).to_string())
+            })?;
             litellm_inference::diagnostic::provider(
-                &prepared.body.model,
-                prepared.provider.as_str(),
+                provider.model,
+                <&'static str>::from(provider.provider),
             );
-            let request = self.prepare_outbound(prepared, &context).boxed().await?;
-            let cache = CallCache::<route::Messages>::from_wire(
-                self.cache.as_ref().filter(|_| request.cacheable()),
-                context.cache,
-                &request.identity,
-                &request.wire,
-            );
-            let identity = request.identity.clone();
-            let config = request.provider.config();
-            let (output, source) = match cache.lookup().await {
-                Some(hit) => hit,
-                None => (
-                    self.call_provider(request, &context).await?,
-                    ResultSource::Provider,
-                ),
-            };
-            let reported = match &output {
-                MessagesCallResponse::Complete(response) => config.reported_cost(response),
-                MessagesCallResponse::Stream { .. } => None,
-            };
-            context
-                .result_ready(ExecutionFacts {
-                    provider: identity,
-                    cost: Cost::settle(&source, reported),
-                    source: source.clone(),
-                })
-                .await?;
-            Ok(cache.finish(output, &source).await)
+            let model = provider.model.to_owned();
+            let secrets = self.secrets.resolve(config.config().secret_names()).await?;
+            handler::execute(self, config, model, call, secrets, &context)
+                .boxed()
+                .await
         })
         .await
     }

litellm-rust/crates/inference-messages/src/prepare.rs

@@ -1,554 +0,0 @@
-use std::time::Duration;
-
-use litellm_auth::SecretValue;
-use litellm_core_utils::{
-    dot_notation_indexing::delete_nested_value,
-    get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
-};
-use litellm_http::request::with_default_headers;
-use litellm_llms::base_llm::{
-    auth::ValidatedEnvironment, messages::context::MessagesTransformContext,
-};
-use litellm_llms_types::formats::messages::MessagesRequest;
-use litellm_secrets::source::SecretSource;
-
-use super::{
-    Error, MessagesCall,
-    common_utils::{MessagesProvider, messages_provider, string_headers},
-    types::invalid_request,
-};
-use litellm_inference::provider::resolve_llm_provider;
-
-struct ResolvedProvider {
-    model: String,
-    provider: MessagesProvider,
-}
-
-pub(super) struct ProviderMessagesRequest {
-    pub(super) provider: MessagesProvider,
-    pub(super) url: String,
-    pub(super) body: MessagesRequest,
-    pub(super) environment: ValidatedEnvironment,
-    pub(super) timeout: Option<Duration>,
-    /// The caller's own credential, reported to the host beside the wire request.
-    pub(super) api_key: Option<SecretValue>,
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) async fn prepare(
-    call: MessagesCall,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderMessagesRequest, Error> {
-    let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
-    let secrets = secrets
-        .resolve(resolved.provider.config().secret_names())
-        .await?;
-    prepare_provider_request(call, resolved, secrets.as_ref())
-}
-
-fn resolve_provider(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<ResolvedProvider, Error> {
-    let resolved = resolve_llm_provider(model, custom_llm_provider, "messages")?;
-    let provider = messages_provider(resolved.provider, resolved.model)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(resolved.provider).to_string()))?;
-    Ok(ResolvedProvider {
-        model: resolved.model.to_string(),
-        provider,
-    })
-}
-
-fn prepare_provider_request(
-    call: MessagesCall,
-    resolved: ResolvedProvider,
-    secrets: &dyn Lookup,
-) -> Result<ProviderMessagesRequest, Error> {
-    let ResolvedProvider { model, provider } = resolved;
-    let MessagesCall {
-        body,
-        api_key,
-        api_base,
-        litellm_params,
-        extra_headers,
-        provider_specific_header,
-        timeout,
-        shaping,
-        ..
-    } = call;
-    let connection_params = litellm_router_types::LitellmParams {
-        api_base: api_base.clone().or(litellm_params.api_base.clone()),
-        ..litellm_params
-    };
-    let config = provider.config();
-    let env_lookup = |key: &str| secrets.get(key);
-
-    let sanitized = config.shape_request(
-        MessagesRequest { model, ..body },
-        shaping.settings.reasoning_auto_summary,
-    )?;
-    let trimmed =
-        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
-    let transformed = config.transform_anthropic_messages_request(
-        trimmed,
-        &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
-    )?;
-
-    let scoped =
-        get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
-    let forwarded = string_headers(Some(
-        extra_headers.into_iter().flatten().chain(scoped).collect(),
-    ))?;
-    let forwarded = config.request_headers(
-        with_default_headers(forwarded, config.default_headers()),
-        &transformed,
-    );
-    let environment = config.validate_environment(
-        forwarded,
-        api_key.as_deref(),
-        &transformed.model,
-        &connection_params,
-        &env_lookup,
-    )?;
-
-    let url = config.get_complete_url(
-        api_base.as_deref(),
-        &transformed.model,
-        &connection_params,
-        transformed.params.stream == Some(true),
-        &env_lookup,
-    )?;
-
-    Ok(ProviderMessagesRequest {
-        provider,
-        url,
-        body: transformed,
-        environment,
-        timeout: timeout.or_else(|| shaping.settings.request_timeout()),
-        api_key: api_key.map(SecretValue::new),
-    })
-}
-
-fn without_additional_drop_params(
-    request: MessagesRequest,
-    paths: &[String],
-) -> Result<MessagesRequest, Error> {
-    if paths.is_empty() {
-        return Ok(request);
-    }
-    let params = serde_json::to_value(request.params).map_err(invalid_request)?;
-    let trimmed = paths
-        .iter()
-        .fold(params, |params, path| delete_nested_value(params, path));
-    Ok(MessagesRequest {
-        params: serde_json::from_value(trimmed).map_err(invalid_request)?,
-        ..request
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use litellm_llms::base_llm::auth::resolve_auth;
-    use litellm_llms_types::headers::ProviderSpecificHeaders;
-    use rstest::{fixture, rstest};
-    use serde_json::{Map, Value, json};
-
-    use super::*;
-    use crate::{MessagesSettings, MessagesShaping};
-
-    #[fixture]
-    fn shaping() -> MessagesShaping {
-        MessagesShaping::default()
-    }
-
-    fn body(value: Value) -> MessagesRequest {
-        serde_json::from_value(value).unwrap()
-    }
-
-    fn prepare(call: MessagesCall) -> Result<ProviderMessagesRequest, Error> {
-        prepare_with_secrets(call, &|_: &str| None)
-    }
-
-    fn prepare_with_secrets(
-        call: MessagesCall,
-        secrets: &dyn Lookup,
-    ) -> Result<ProviderMessagesRequest, Error> {
-        let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
-        prepare_provider_request(call, resolved, secrets)
-    }
-
-    /// The headers as they go on the wire, credential applied.
-    fn wire_headers(prepared: &ProviderMessagesRequest) -> Vec<(String, String)> {
-        tokio::runtime::Builder::new_current_thread()
-            .build()
-            .unwrap()
-            .block_on(resolve_auth(
-                &litellm_auth::AuthServices::default(),
-                prepared.environment.clone(),
-                &|_| None,
-            ))
-            .unwrap()
-            .headers
-    }
-
-    #[rstest]
-    #[case::api_key(
-        &[("ANTHROPIC_API_KEY", "sk-secret")],
-        &[("x-api-key", "sk-secret")],
-        "https://api.anthropic.com/v1/messages"
-    )]
-    #[case::auth_token(
-        &[("ANTHROPIC_AUTH_TOKEN", "token")],
-        &[("authorization", "Bearer token")],
-        "https://api.anthropic.com/v1/messages"
-    )]
-    #[case::api_base(
-        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
-        &[("x-api-key", "sk-secret")],
-        "https://gateway.test/v1/messages"
-    )]
-    #[case::sdk_base_url(
-        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
-        &[("x-api-key", "sk-secret")],
-        "https://sdk.test/v1/messages"
-    )]
-    fn credentials_and_base_come_from_the_resolved_secrets(
-        shaping: MessagesShaping,
-        #[case] secrets: &[(&str, &str)],
-        #[case] expected_auth: &[(&str, &str)],
-        #[case] expected_url: &str,
-    ) {
-        let lookup = |name: &str| {
-            secrets
-                .iter()
-                .find(|(key, _)| *key == name)
-                .map(|(_, value)| value.to_string())
-        };
-        let prepared = prepare_with_secrets(
-            MessagesCall {
-                body: body(
-                    json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
-                ),
-                api_key: None,
-                api_base: None,
-                custom_llm_provider: Some("anthropic".into()),
-                litellm_params: Default::default(),
-                extra_headers: None,
-                provider_specific_header: None,
-                timeout: None,
-                shaping,
-            },
-            &lookup,
-        )
-        .unwrap();
-        let headers = wire_headers(&prepared);
-        let auth: Vec<(&str, &str)> = headers
-            .iter()
-            .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
-            .map(|(name, value)| (name.as_str(), value.as_str()))
-            .collect();
-        assert_eq!(
-            (auth.as_slice(), prepared.url.as_str()),
-            (expected_auth, expected_url)
-        );
-    }
-
-    #[rstest]
-    #[case::nothing_configured(None, None, None)]
-    #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
-    #[case::call_timeout_wins(
-        Some(Duration::from_secs(30)),
-        Some(1800.0),
-        Some(Duration::from_secs(30))
-    )]
-    #[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
-    #[case::unusable_setting_ignored(None, Some(-1.0), None)]
-    #[case::zero_setting_ignored(None, Some(0.0), None)]
-    fn timeout_falls_back_to_the_configured_request_timeout(
-        shaping: MessagesShaping,
-        #[case] call_timeout: Option<Duration>,
-        #[case] request_timeout: Option<f64>,
-        #[case] expected: Option<Duration>,
-    ) {
-        let prepared = prepare(MessagesCall {
-            body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://anthropic.test".into()),
-            custom_llm_provider: Some("anthropic".into()),
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: call_timeout,
-            shaping: MessagesShaping {
-                settings: MessagesSettings { request_timeout, ..shaping.settings },
-                ..shaping
-            },
-        })
-        .unwrap();
-        assert_eq!(prepared.timeout, expected);
-    }
-
-    fn prepared_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
-        prepare(MessagesCall {
-            body: body(fields),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://anthropic.test".into()),
-            custom_llm_provider: Some("anthropic".into()),
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: None,
-            shaping,
-        })
-        .map(|prepared| serde_json::to_value(prepared.body).unwrap())
-    }
-
-    #[rstest]
-    #[case::top_level_and_nested_paths(
-        json!({
-            "max_tokens": 1024,
-            "thinking": {"type": "enabled", "budget_tokens": 2048},
-            "context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
-            "metadata": {"user_id": "u1"},
-            "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
-        }),
-        &["thinking", "context_management", "tools[*].input_examples"],
-        json!({
-            "max_tokens": 1024,
-            "metadata": {"user_id": "u1"},
-            "tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
-        }),
-    )]
-    #[case::no_paths(
-        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
-        &[],
-        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
-    )]
-    #[case::model_and_messages_are_never_dropped(
-        json!({"max_tokens": 16}),
-        &["model", "messages", "messages[0].content"],
-        json!({"max_tokens": 16}),
-    )]
-    fn prepared_body_drops_configured_paths(
-        shaping: MessagesShaping,
-        #[case] fields: Value,
-        #[case] additional_drop_params: &[&str],
-        #[case] expected_fields: Value,
-    ) {
-        let with_messages = |fields: Value| -> Value {
-            let Value::Object(fields) = fields else {
-                unreachable!()
-            };
-            Value::Object(
-                [
-                    ("model".to_string(), json!("claude-test")),
-                    (
-                        "messages".to_string(),
-                        json!([{"role": "user", "content": "hi"}]),
-                    ),
-                ]
-                .into_iter()
-                .chain(fields)
-                .collect(),
-            )
-        };
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                additional_drop_params: additional_drop_params
-                    .iter()
-                    .map(ToString::to_string)
-                    .collect(),
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert_eq!(
-            prepared_body(with_messages(fields), shaping),
-            Ok(with_messages(expected_fields))
-        );
-    }
-
-    #[rstest]
-    #[case::model_prefix_picks_the_provider(
-        "azure_ai/claude-test",
-        None,
-        &[("x-priority", "extra"), ("x-scoped", "azure_ai")]
-    )]
-    #[case::explicit_provider(
-        "claude-test",
-        Some("anthropic"),
-        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
-    )]
-    #[case::provider_prefix_on_an_anthropic_model(
-        "anthropic/claude-test",
-        None,
-        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
-    )]
-    fn provider_specific_headers_follow_the_resolved_provider(
-        shaping: MessagesShaping,
-        #[case] model: &str,
-        #[case] custom_llm_provider: Option<&str>,
-        #[case] expected: &[(&str, &str)],
-    ) {
-        let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
-            {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
-            {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
-        ]))
-        .unwrap();
-        let prepared = prepare(MessagesCall {
-            body: body(
-                json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
-            ),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://resource.services.ai.azure.com".into()),
-            custom_llm_provider: custom_llm_provider.map(Into::into),
-            litellm_params: Default::default(),
-            extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
-            provider_specific_header: Some(configured),
-            timeout: None,
-            shaping,
-        })
-        .unwrap();
-        let caller_headers: Vec<(&str, &str)> = prepared
-            .environment
-            .headers
-            .iter()
-            .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
-            .map(|(name, value)| (name.as_str(), value.as_str()))
-            .collect();
-        assert_eq!(caller_headers, expected);
-    }
-
-    #[rstest]
-    fn prepared_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "anthropic/claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16
-                }),
-                shaping,
-            ),
-            Ok(json!({
-                "model": "claude-test",
-                "messages": [{"role": "user", "content": "hi"}],
-                "max_tokens": 16
-            }))
-        );
-    }
-
-    #[rstest]
-    fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                reasoning_auto_summary: true,
-                additional_drop_params: vec!["thinking.display".to_string()],
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 4096,
-                    "thinking": {"type": "enabled", "budget_tokens": 2048}
-                }),
-                shaping,
-            ),
-            Ok(json!({
-                "model": "claude-test",
-                "messages": [{"role": "user", "content": "hi"}],
-                "max_tokens": 4096,
-                "thinking": {"type": "enabled", "budget_tokens": 2048}
-            }))
-        );
-    }
-
-    #[rstest]
-    fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(shaping: MessagesShaping) {
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                additional_drop_params: vec!["metadata.user_id".to_string()],
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert!(matches!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16,
-                    "metadata": {"user_id": 123}
-                }),
-                shaping,
-            ),
-            Err(Error::InvalidRequest(_))
-        ));
-    }
-
-    #[rstest]
-    fn prepared_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16,
-                    "metadata": {"user_id": 123}
-                }),
-                shaping,
-            ),
-            Err(Error::InvalidRequest(
-                litellm_llms::ErrorDetail::InvalidValue {
-                    field: "metadata.user_id",
-                    expected: "a string",
-                    actual: json!(123),
-                }
-            ))
-        );
-    }
-    #[rstest]
-    #[case::mantle(
-        "bedrock_mantle/claude-test",
-        "https://bedrock-mantle.us-west-2.api.aws",
-        "/anthropic/v1/messages",
-        "bedrock"
-    )]
-    #[case::platform(
-        "bedrock/claude_platform/claude-test",
-        "https://aws-external-anthropic.us-west-2.api.aws",
-        "/v1/messages",
-        "aws-external-anthropic"
-    )]
-    fn explicit_endpoint_region_is_used_by_the_signer(
-        shaping: MessagesShaping,
-        #[case] model: &str,
-        #[case] base: &str,
-        #[case] path: &str,
-        #[case] service: &str,
-    ) {
-        let prepared = prepare_with_secrets(MessagesCall {
-            body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
-            api_key: None,
-            api_base: Some(base.into()),
-            custom_llm_provider: None,
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: None,
-            shaping,
-        }, &|name: &str| match name {
-            "AWS_REGION_NAME" => Some("us-east-1".into()),
-            "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
-            _ => None,
-        }).unwrap();
-        assert_eq!(prepared.url, format!("{base}{path}"));
-        assert!(
-            matches!(prepared.environment.auth, litellm_llms::base_llm::auth::AuthScheme::AwsSigV4 { region, service: actual, .. } if region == "us-west-2" && actual == service)
-        );
-    }
-}

litellm-rust/crates/inference-ocr/src/arguments.rs

@@ -44,8 +44,8 @@ pub fn consumed_optional_param_names(
     model: &str,
     custom_llm_provider: Option<&str>,
 ) -> Result<Vec<&'static str>, Error> {
-    let (model, config) = resolve_provider_config(model, custom_llm_provider)?;
-    let provider_fields = config.get_supported_ocr_params(&model);
+    let (provider, config) = resolve_provider_config(model, custom_llm_provider)?;
+    let provider_fields = config.get_supported_ocr_params(provider.model);
     let auth_fields: &[&str] = match config {
         OcrConfigKind::AwsTextract | OcrConfigKind::AwsTextractAnalyze => AWS_AUTH_OPTION_FIELDS,
         OcrConfigKind::AzureAi

litellm-rust/crates/inference-ocr/src/client.rs

@@ -1,81 +0,0 @@
-use litellm_host::observation::ObservationSender;
-use std::sync::Arc;
-
-use litellm_host::interceptors::Interceptors;
-use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
-use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
-
-use super::{
-    handler::perform_ocr_request,
-    types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest},
-};
-
-#[derive(Clone)]
-pub struct OcrRoute {
-    client: OcrClient,
-}
-
-impl OcrRoute {
-    pub fn new(client: OcrClient) -> Self {
-        Self { client }
-    }
-
-    pub async fn execute(
-        &self,
-        request: LiteLLMOcrRequest,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<ObservationSender>,
-    ) -> Result<LiteLLMOcrResponse, Error> {
-        litellm_host::lifecycle::observe_unary(
-            observers.clone(),
-            self.run(request, interceptors, observers.as_ref()),
-        )
-        .await
-    }
-
-    #[tracing::instrument(name = "litellm.route", skip_all, fields(
-        route = "ocr",
-        model = %request.model,
-        resolved_model = %request.model,
-        provider = <&str>::from(request.config.provider()),
-        stream = false,
-        outcome
-    ))]
-    pub(super) async fn run(
-        &self,
-        request: LiteLLMOcrRequest,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<LiteLLMOcrResponse, Error> {
-        litellm_inference::diagnostic::unary(async {
-            let caller_document = matches!(&request.document, OcrDocumentInput::Document(_));
-            let prepared = prepare_request_document(request).await?;
-            let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
-                Box::pin(perform_ocr_request(
-                    &self.client,
-                    prepared,
-                    interceptors,
-                    caller_document,
-                    observers,
-                ));
-            execute.await
-        })
-        .await
-    }
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-async fn prepare_request_document(
-    request: LiteLLMOcrRequest<OcrDocumentInput>,
-) -> Result<ResolvedOcrRequest, Error> {
-    if let OcrDocumentInput::Document(_) = &request.document {
-        return request.map_document(super::document::prepare_document);
-    }
-    let logger = litellm_tracing::Logger::current();
-    let span = tracing::Span::current();
-    tokio::task::spawn_blocking(move || {
-        logger.scope(|| span.in_scope(|| request.map_document(super::document::prepare_document)))
-    })
-    .await
-    .map_err(|error| Error::DocumentTask(Arc::new(error)))?
-}

litellm-rust/crates/inference-ocr/src/document.rs

@@ -1,10 +1,10 @@
-use std::{collections::BTreeMap as Map, io::Read, path::Path};
+use std::{collections::BTreeMap as Map, io::Read, path::Path, sync::Arc};
 
 use base64::{Engine, engine::general_purpose::STANDARD};
 use litellm_llms::base_llm::ocr::{error::Error, transformation::OCR_INLINE_MAX_BYTES};
 use litellm_llms_types::formats::ocr::OcrDocument;
 
-use crate::types::OcrDocumentInput;
+use crate::types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest};
 
 pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
     match input {
@@ -24,6 +24,21 @@ pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
     }
 }
 
+pub(crate) async fn resolve_document(
+    request: LiteLLMOcrRequest<OcrDocumentInput>,
+) -> Result<ResolvedOcrRequest, Error> {
+    if let OcrDocumentInput::Document(_) = &request.document {
+        return request.map_document(prepare_document);
+    }
+    let logger = litellm_tracing::Logger::current();
+    let span = tracing::Span::current();
+    tokio::task::spawn_blocking(move || {
+        logger.scope(|| span.in_scope(|| request.map_document(prepare_document)))
+    })
+    .await
+    .map_err(|error| Error::DocumentTask(Arc::new(error)))?
+}
+
 pub fn read_path_document(path: &Path, mime_type: Option<&str>) -> Result<OcrDocument, Error> {
     let mut bytes = Vec::new();
     std::fs::File::open(path)

litellm-rust/crates/inference-ocr/src/handler.rs

@@ -4,29 +4,45 @@ use litellm_host::{lifecycle::ExecutionEvent, observation::ObservationSender};
 use litellm_llms::base_llm::ocr::{
     error::Error,
     handler::{CallHooks, OcrClient},
-    transformation::PreparedOcrRequest,
+    transformation::{OcrConnection, PreparedOcrRequest},
 };
 use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
-use super::{arguments::is_secret_param, prepare::prepare_request, provider_config::OcrConfigKind};
-use crate::types::ResolvedOcrRequest;
+use super::{arguments::is_secret_param, provider_config::OcrConfigKind};
+use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
 
-pub(crate) async fn perform_ocr_request(
+pub(crate) async fn execute(
     client: &OcrClient,
-    request: ResolvedOcrRequest,
+    config: OcrConfigKind,
+    call: ResolvedOcrRequest,
+    secrets: Secrets,
     host: &impl Interceptors<Error>,
     caller_document: bool,
     observers: Option<&ObservationSender>,
 ) -> Result<LiteLLMOcrResponse, Error> {
-    request.response_format()?;
-    let config = request.config;
-    let secrets = client
-        .secret_source()
-        .resolve(&config.secret_names())
-        .await
-        .map_err(|error| Error::Secret(std::sync::Arc::new(error)))?;
-    let request = prepare_request(request, caller_document, client, secrets);
+    call.response_format()?;
+    let LiteLLMOcrRequest {
+        model,
+        document,
+        credentials,
+        transport,
+        optional_params,
+        input_sources,
+        azure_ad_token_provider,
+        ..
+    } = call;
+    let resolved = config.resolve_credentials(credentials, &secrets);
+    let request = PreparedOcrRequest {
+        model,
+        document,
+        connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
+        caller_document,
+        optional_params,
+        input_sources,
+        azure_ad_token_provider,
+    };
     let interceptors = OcrCallHooks::new(host, &request, config, observers);
     config.ocr(client, &request, &interceptors).await
 }
@@ -86,3 +102,257 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
         Box::pin(self.interceptors.after_provider_response(raw))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
+    use litellm_llms_types::formats::ocr::OcrResponseFormat;
+    use rstest::rstest;
+    use serde_json::{Value, json};
+    use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
+
+    use super::*;
+    use crate::{
+        OcrRoute,
+        types::LiteLLMOcrRequest,
+        wire::{OcrWireRequest, decode_request},
+    };
+
+    fn client() -> OcrClient {
+        OcrClient::for_test(
+            litellm_http::Client::plain_for_test(),
+            litellm_http::Client::no_redirect_for_test(),
+        )
+        .with_secrets(litellm_inference_testing::no_secrets())
+    }
+
+    fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
+        decode_request(OcrWireRequest {
+            model: model.into(),
+            document,
+            api_key: Some(litellm_auth::SecretValue::new("test-key")),
+            api_base: Some(base.into()),
+            custom_llm_provider: None,
+            extra_headers: None,
+            optional_params: options.as_object().unwrap().clone(),
+            input_sources: Default::default(),
+            timeout_seconds: Some(2.0),
+        })
+        .unwrap()
+    }
+
+    async fn run_request(request: LiteLLMOcrRequest) -> Result<LiteLLMOcrResponse, Error> {
+        OcrRoute::new(client()).execute(request, &(), None).await
+    }
+
+    async fn upstream(request_count: u64, response: Value) -> MockServer {
+        let server = MockServer::start().await;
+        Mock::given(any())
+            .respond_with(ResponseTemplate::new(200).set_body_json(response))
+            .up_to_n_times(request_count)
+            .mount(&server)
+            .await;
+        server
+    }
+
+    fn image(url: &str) -> Value {
+        json!({"type": "image_url", "image_url": url})
+    }
+
+    fn body(request: &wiremock::Request) -> Value {
+        serde_json::from_slice(&request.body).unwrap()
+    }
+
+    fn assert_mistral_request(request: &wiremock::Request) {
+        assert_eq!(
+            request
+                .headers
+                .get("authorization")
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "Bearer test-key"
+        );
+        assert_eq!(
+            request
+                .headers
+                .get("content-type")
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "application/json"
+        );
+        assert_eq!(
+            body(request),
+            json!({
+                "model": "mistral-ocr-maas",
+                "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
+                "pages": [0, 2],
+                "include_image_base64": true,
+                "unknown": "preserved"
+            })
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
+        let upstream = upstream(
+            1,
+            json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
+        )
+        .await;
+        let request = request(
+            "cohere/parse",
+            &upstream.uri(),
+            image("https://example.com/original.png"),
+            json!({
+                "output_format": "markdown", "timeout": 30,
+                "extra_body": {
+                    "output_format": {"future": true},
+                    "document": {"type": "image_url", "image_url": "https://example.com/a.png",
+                        "provider_options": {"nested": [false, 0, null]}}
+                }
+            }),
+        );
+
+        run_request(request).await.unwrap();
+
+        let [request] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        assert_eq!(
+            body(&request),
+            json!({
+                "model": "parse", "output_format": {"future": true},
+                "document": {"type": "image_url", "image_url": "https://example.com/a.png",
+                    "provider_options": {"nested": [false, 0, null]}}
+            })
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn explicit_null_options_use_defaults_before_http() {
+        let upstream = upstream(
+            1,
+            json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
+        )
+        .await;
+        let request = request(
+            "cohere/parse",
+            &upstream.uri(),
+            image("https://example.com/a.png"),
+            json!({"output_format": null, "req_format": null}),
+        );
+        assert_eq!(
+            request.response_format().unwrap(),
+            OcrResponseFormat::Litellm
+        );
+
+        run_request(request).await.unwrap();
+
+        let [request] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        let body = body(&request);
+        assert_eq!(body["output_format"], "markdown");
+        assert!(body.get("req_format").is_none());
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
+        let upstream = upstream(
+            2,
+            json!({
+                "pages": [{"index": 0, "markdown": "hello"}],
+                "extra": "preserved"
+            }),
+        )
+        .await;
+        let options = json!({
+            "pages": [0, 2],
+            "include_image_base64": true,
+            "vertex_project": "project-1",
+            "vertex_location": "us-central1",
+            "unknown": "preserved"
+        });
+        let document =
+            json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
+        let direct_call = request(
+            "mistral/mistral-ocr-maas",
+            &upstream.uri(),
+            document.clone(),
+            options.clone(),
+        );
+        let vertex_call = request(
+            "vertex_ai/mistral-ocr-maas",
+            &upstream.uri(),
+            document,
+            options,
+        );
+        assert_eq!(direct_call.transport.timeout, Some(Duration::from_secs(2)));
+        assert_eq!(vertex_call.transport.timeout, Some(Duration::from_secs(2)));
+        let direct_response = run_request(direct_call).await.unwrap().into_json();
+        let vertex_response = run_request(vertex_call).await.unwrap().into_json();
+
+        let [direct_http, vertex_http] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        assert_eq!(direct_http.url.path(), "/v1/ocr");
+        assert_eq!(
+            vertex_http.url.path(),
+            "/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
+        );
+        assert_mistral_request(&direct_http);
+        assert_mistral_request(&vertex_http);
+        assert_eq!(direct_response, vertex_response);
+        assert_eq!(direct_response["model"], "mistral-ocr-maas");
+        assert_eq!(direct_response["object"], "ocr");
+        assert_eq!(direct_response["extra"], "preserved");
+    }
+
+    #[derive(serde::Deserialize)]
+    struct KnownParams {
+        pages: Option<Vec<i64>>,
+    }
+
+    #[rstest]
+    fn parsed_provider_params_separates_known_and_extra_params() {
+        let arguments: CallArguments = serde_json::from_value(json!({
+            "pages": [0, 2],
+            "future_ocr_option": true,
+            "extra_body": {"provider_option": "value"}
+        }))
+        .unwrap();
+        let known: KnownParams = parse_options(&arguments).unwrap();
+        assert_eq!(known.pages, Some(vec![0, 2]));
+        assert_eq!(arguments["future_ocr_option"], true);
+        assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
+        assert_eq!(
+            arguments
+                .iter()
+                .filter(|(name, _)| name.as_str() != "pages")
+                .count(),
+            2
+        );
+        assert_eq!(
+            compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
+            json!({
+                "pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
+            })
+        );
+    }
+}

litellm-rust/crates/inference-ocr/src/lib.rs

@@ -1,10 +1,82 @@
 pub mod arguments;
-mod client;
-pub use client::OcrRoute;
 pub mod document;
 pub(crate) mod handler;
-pub(crate) mod prepare;
 pub mod provider_config;
 pub mod route;
 pub mod types;
 pub mod wire;
+
+use std::sync::Arc;
+
+use litellm_host::interceptors::Interceptors;
+use litellm_host::observation::ObservationSender;
+use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
+use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
+
+use crate::{
+    handler::execute,
+    types::{LiteLLMOcrRequest, OcrDocumentInput},
+};
+
+#[derive(Clone)]
+pub struct OcrRoute {
+    client: OcrClient,
+}
+
+impl OcrRoute {
+    pub fn new(client: OcrClient) -> Self {
+        Self { client }
+    }
+
+    pub async fn execute(
+        &self,
+        request: LiteLLMOcrRequest,
+        interceptors: &impl Interceptors<Error>,
+        observers: Option<ObservationSender>,
+    ) -> Result<LiteLLMOcrResponse, Error> {
+        litellm_host::lifecycle::observe_unary(
+            observers.clone(),
+            self.run(request, interceptors, observers.as_ref()),
+        )
+        .await
+    }
+
+    #[tracing::instrument(name = "litellm.route", skip_all, fields(
+        route = "ocr",
+        model = %call.model,
+        resolved_model = %call.model,
+        provider = <&str>::from(call.config.provider()),
+        stream = false,
+        outcome
+    ))]
+    async fn run(
+        &self,
+        call: LiteLLMOcrRequest,
+        interceptors: &impl Interceptors<Error>,
+        observers: Option<&ObservationSender>,
+    ) -> Result<LiteLLMOcrResponse, Error> {
+        litellm_inference::diagnostic::unary(async {
+            let caller_document = matches!(&call.document, OcrDocumentInput::Document(_));
+            let call = document::resolve_document(call).await?;
+            let config = call.config;
+            let secrets = self
+                .client
+                .secret_source()
+                .resolve(&config.secret_names())
+                .await
+                .map_err(|error| Error::Secret(Arc::new(error)))?;
+            let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
+                Box::pin(execute(
+                    &self.client,
+                    config,
+                    call,
+                    secrets,
+                    interceptors,
+                    caller_document,
+                    observers,
+                ));
+            execute.await
+        })
+        .await
+    }
+}

litellm-rust/crates/inference-ocr/src/prepare.rs

@@ -1,309 +0,0 @@
-use litellm_auth::{InputSource, SecretValue, Sourced};
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_llms::base_llm::ocr::{
-    handler::OcrClient,
-    transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
-};
-use litellm_secrets::source::Secrets;
-
-use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
-
-pub(crate) fn prepare_request(
-    request: ResolvedOcrRequest,
-    caller_document: bool,
-    client: &OcrClient,
-    secrets: Secrets,
-) -> PreparedOcrRequest {
-    let credentials = request.credentials.clone();
-    let (preferred_api_key_env, api_base_env) = match request.config.provider() {
-        LlmProviders::Mistral => (
-            Some("MISTRAL_AZURE_API_KEY"),
-            Some("MISTRAL_AZURE_API_BASE"),
-        ),
-        LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
-        _ => (None, None),
-    };
-    let secret = |name: &str| secrets.truthy(name);
-    let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
-        credentials.api_key.clone().or_else(|| {
-            preferred_api_key_env
-                .into_iter()
-                .chain(request.config.get_api_key_env_var())
-                .find_map(secret)
-                .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
-        })
-    });
-    let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
-        credentials.api_base.clone().or_else(|| {
-            api_base_env
-                .and_then(secret)
-                .map(|value| Sourced::new(value, InputSource::Environment))
-        })
-    });
-    let resolved = request
-        .config
-        .resolve_connection_params(OcrCredentialInputs {
-            dynamic_api_key,
-            dynamic_api_base,
-            ..credentials
-        });
-    let LiteLLMOcrRequest {
-        model,
-        document,
-        transport,
-        optional_params,
-        input_sources,
-        azure_ad_token_provider,
-        ..
-    } = request;
-    PreparedOcrRequest {
-        model,
-        document,
-        connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
-        caller_document,
-        optional_params,
-        input_sources,
-        azure_ad_token_provider,
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::time::Duration;
-
-    use futures_util::future::BoxFuture;
-    use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
-    use litellm_host::interceptors::WireRequest;
-    use litellm_llms::{
-        base_llm::ocr::{
-            error::Error,
-            handler::{CallHooks, OcrClient},
-            transformation::BaseOcrConfig,
-        },
-        cohere::ocr::transformation::CohereParseConfig,
-        mistral::ocr::transformation::MistralOcrConfig,
-        vertex_ai::ocr::transformation::VertexAiOcrConfig,
-    };
-    use litellm_llms_types::formats::ocr::OcrResponseFormat;
-    use serde_json::{Value, json};
-
-    use super::*;
-    use crate::{
-        document::prepare_document,
-        types::LiteLLMOcrRequest,
-        wire::{OcrWireRequest, decode_request},
-    };
-
-    /// Stands in for a host with no interceptors registered.
-    struct NoHooks;
-
-    impl CallHooks<Error> for NoHooks {
-        fn before_provider_request(
-            &self,
-            wire: WireRequest,
-        ) -> BoxFuture<'_, Result<WireRequest, Error>> {
-            Box::pin(async move { Ok(wire) })
-        }
-
-        fn response_received<'a>(&'a self, _body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
-            Box::pin(async { Ok(()) })
-        }
-    }
-
-    fn client() -> OcrClient {
-        OcrClient::for_test(
-            litellm_http::Client::plain_for_test(),
-            litellm_http::Client::no_redirect_for_test(),
-        )
-    }
-
-    fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
-        decode_request(OcrWireRequest {
-            model: model.into(),
-            document,
-            api_key: Some(litellm_auth::SecretValue::new("test-key")),
-            api_base: Some(base.into()),
-            custom_llm_provider: None,
-            extra_headers: None,
-            optional_params: options.as_object().unwrap().clone(),
-            input_sources: Default::default(),
-            timeout_seconds: Some(2.0),
-        })
-        .unwrap()
-    }
-
-    fn prepared(request: LiteLLMOcrRequest) -> PreparedOcrRequest {
-        prepare_request(
-            request.map_document(prepare_document).unwrap(),
-            true,
-            &client(),
-            std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
-        )
-    }
-
-    fn image(url: &str) -> Value {
-        json!({"type": "image_url", "image_url": url})
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
-        let request = request(
-            "cohere/parse",
-            "https://example.com",
-            image("https://example.com/original.png"),
-            json!({
-                "output_format": "markdown", "timeout": 30,
-                "extra_body": {
-                    "output_format": {"future": true},
-                    "document": {"type": "image_url", "image_url": "https://example.com/a.png",
-                        "provider_options": {"nested": [false, 0, null]}}
-                }
-            }),
-        );
-
-        let http = CohereParseConfig
-            .prepare_request(&prepared(request), &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        let body: Value = serde_json::from_slice(http.body()).unwrap();
-        assert_eq!(
-            body,
-            json!({
-                "model": "parse", "output_format": {"future": true},
-                "document": {"type": "image_url", "image_url": "https://example.com/a.png",
-                    "provider_options": {"nested": [false, 0, null]}}
-            })
-        );
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn explicit_null_options_use_defaults_before_http() {
-        let request = request(
-            "cohere/parse",
-            "https://example.com",
-            image("https://example.com/a.png"),
-            json!({"output_format": null, "req_format": null}),
-        );
-        assert_eq!(
-            request.response_format().unwrap(),
-            OcrResponseFormat::Litellm
-        );
-
-        let http = CohereParseConfig
-            .prepare_request(&prepared(request), &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        let body: Value = serde_json::from_slice(http.body()).unwrap();
-        assert_eq!(body["output_format"], "markdown");
-        assert!(body.get("req_format").is_none());
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
-        let options = json!({
-            "pages": [0, 2],
-            "include_image_base64": true,
-            "vertex_project": "project-1",
-            "vertex_location": "us-central1",
-            "unknown": "preserved"
-        });
-        let document =
-            json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
-        let direct = prepared(request(
-            "mistral/mistral-ocr-maas",
-            "https://mistral.test",
-            document.clone(),
-            options.clone(),
-        ));
-        let vertex = prepared(request(
-            "vertex_ai/mistral-ocr-maas",
-            "https://vertex.test",
-            document,
-            options,
-        ));
-
-        let direct_http = MistralOcrConfig
-            .prepare_request(&direct, &client(), &NoHooks)
-            .await
-            .unwrap();
-        let vertex_http = VertexAiOcrConfig
-            .prepare_request(&vertex, &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        assert_eq!(direct_http.url(), "https://mistral.test/v1/ocr");
-        assert_eq!(
-            vertex_http.url(),
-            "https://vertex.test/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
-        );
-        for http in [&direct_http, &vertex_http] {
-            assert_eq!(http.header("authorization").unwrap(), "Bearer test-key");
-            assert_eq!(http.header("content-type").unwrap(), "application/json");
-            assert_eq!(http.timeout(), Some(Duration::from_secs(2)));
-            let body: Value = serde_json::from_slice(http.body()).unwrap();
-            assert_eq!(
-                body,
-                json!({
-                    "model": "mistral-ocr-maas",
-                    "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
-                    "pages": [0, 2],
-                    "include_image_base64": true,
-                    "unknown": "preserved"
-                })
-            );
-        }
-        let payload = serde_json::to_vec(
-            &json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}),
-        )
-        .unwrap();
-        let direct_response = MistralOcrConfig
-            .transform_ocr_response(&direct.model, &payload, OcrResponseFormat::Litellm)
-            .unwrap()
-            .into_json();
-        let vertex_response = VertexAiOcrConfig
-            .transform_ocr_response(&vertex.model, &payload, OcrResponseFormat::Litellm)
-            .unwrap()
-            .into_json();
-        assert_eq!(direct_response, vertex_response);
-        assert_eq!(direct_response["model"], "mistral-ocr-maas");
-        assert_eq!(direct_response["object"], "ocr");
-        assert_eq!(direct_response["extra"], "preserved");
-    }
-
-    #[derive(serde::Deserialize)]
-    struct KnownParams {
-        pages: Option<Vec<i64>>,
-    }
-
-    #[rstest::rstest]
-    fn parsed_provider_params_separates_known_and_extra_params() {
-        let arguments: CallArguments = serde_json::from_value(json!({
-            "pages": [0, 2],
-            "future_ocr_option": true,
-            "extra_body": {"provider_option": "value"}
-        }))
-        .unwrap();
-        let known: KnownParams = parse_options(&arguments).unwrap();
-        assert_eq!(known.pages, Some(vec![0, 2]));
-        assert_eq!(arguments["future_ocr_option"], true);
-        assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
-        assert_eq!(
-            arguments
-                .iter()
-                .filter(|(name, _)| name.as_str() != "pages")
-                .count(),
-            2
-        );
-        assert_eq!(
-            compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
-            json!({
-                "pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
-            })
-        );
-    }
-}

litellm-rust/crates/inference-ocr/src/provider_config.rs

@@ -1,5 +1,6 @@
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
+use litellm_auth::{InputSource, SecretValue, Sourced};
+use litellm_core_utils::get_llm_provider_logic::{LlmProviders, get_custom_llm_provider};
+use litellm_inference::provider::{ResolvedProvider, resolve_llm_provider};
 use litellm_llms::{
     aws_textract::ocr::{
         analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation,
@@ -25,6 +26,7 @@ use litellm_llms::{
     },
 };
 use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat};
+use litellm_secrets::source::Secrets;
 
 macro_rules! with_config {
     ($kind:expr, $config:ident => $body:expr) => {
@@ -115,7 +117,18 @@ impl OcrConfigKind {
     }
 
     pub(crate) fn secret_names(self) -> Vec<&'static str> {
-        with_config!(self, config => config.secret_names())
+        let names = with_config!(self, config => config.secret_names());
+        let fallback_names: &[&str] = match self.provider() {
+            LlmProviders::Mistral => &["MISTRAL_AZURE_API_KEY", "MISTRAL_AZURE_API_BASE"],
+            LlmProviders::AzureAi => &["AZURE_AI_API_BASE"],
+            _ => &[],
+        };
+        let additional_names = fallback_names
+            .iter()
+            .copied()
+            .filter(|name| !names.contains(name))
+            .collect::<Vec<_>>();
+        names.into_iter().chain(additional_names).collect()
     }
 
     pub(crate) fn get_health_check_document(self) -> OcrDocument {
@@ -129,6 +142,43 @@ impl OcrConfigKind {
         with_config!(self, config => config.resolve_connection_params(inputs))
     }
 
+    pub(crate) fn resolve_credentials(
+        self,
+        credentials: OcrCredentialInputs,
+        secrets: &Secrets,
+    ) -> ResolvedOcrCredentials {
+        let (preferred_api_key_env, api_base_env) = match self.provider() {
+            LlmProviders::Mistral => (
+                Some("MISTRAL_AZURE_API_KEY"),
+                Some("MISTRAL_AZURE_API_BASE"),
+            ),
+            LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
+            _ => (None, None),
+        };
+        let secret = |name: &str| secrets.truthy(name);
+        let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
+            credentials.api_key.clone().or_else(|| {
+                preferred_api_key_env
+                    .into_iter()
+                    .chain(self.get_api_key_env_var())
+                    .find_map(secret)
+                    .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
+            })
+        });
+        let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
+            credentials.api_base.clone().or_else(|| {
+                api_base_env
+                    .and_then(secret)
+                    .map(|value| Sourced::new(value, InputSource::Environment))
+            })
+        });
+        self.resolve_connection_params(OcrCredentialInputs {
+            dynamic_api_key,
+            dynamic_api_base,
+            ..credentials
+        })
+    }
+
     pub(crate) async fn ocr(
         self,
         client: &OcrClient,
@@ -164,7 +214,7 @@ pub fn passthrough_response(
     endpoint: &str,
     body: &[u8],
 ) -> Result<Option<LiteLLMOcrResponse>, Error> {
-    let (model, config) = resolve_provider_config(model, Some("azure_ai"))?;
+    let (provider, config) = resolve_provider_config(model, Some("azure_ai"))?;
     let segments: Vec<&str> = endpoint
         .split('/')
         .filter(|segment| !segment.is_empty())
@@ -173,7 +223,7 @@ pub fn passthrough_response(
         OcrConfigKind::AzureAi => segments == AZURE_AI_OCR_PATH,
         OcrConfigKind::AzureCohere => segments == AZURE_COHERE_PARSE_PATH,
         OcrConfigKind::AzureDocumentIntelligence => {
-            segments == AzureDocumentIntelligenceOcrConfig::analyze_path(&model)?
+            segments == AzureDocumentIntelligenceOcrConfig::analyze_path(provider.model)?
         }
         other => {
             let provider: &'static str = other.provider().into();
@@ -183,24 +233,28 @@ pub fn passthrough_response(
     if !is_ocr_endpoint {
         return Ok(None);
     }
-    with_config!(config, config => config.transform_ocr_response(&model, body, OcrResponseFormat::Litellm))
+    with_config!(config, config => config.transform_ocr_response(provider.model, body, OcrResponseFormat::Litellm))
         .map(Some)
 }
 
-pub(crate) fn resolve_provider_config(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<(String, OcrConfigKind), Error> {
-    let provider =
-        get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider {
-            model,
-            custom_llm_provider: LlmProviders::Mistral.into(),
-        });
-    let llm_provider = provider
-        .custom_llm_provider
-        .parse::<LlmProviders>()
-        .map_err(|_| Error::InvalidProvider(provider.custom_llm_provider.to_string()))?;
-    let config = match llm_provider {
+pub(crate) fn resolve_provider_config<'a>(
+    model: &'a str,
+    custom_llm_provider: Option<&'a str>,
+) -> Result<(ResolvedProvider<'a>, OcrConfigKind), Error> {
+    let provider = match resolve_llm_provider(model, custom_llm_provider, "ocr") {
+        Ok(provider) => provider,
+        Err(_) if get_custom_llm_provider(model, custom_llm_provider).is_none() => {
+            ResolvedProvider {
+                model,
+                provider: LlmProviders::Mistral,
+            }
+        }
+        Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
+            return Err(Error::InvalidProvider(provider));
+        }
+        Err(error) => return Err(Error::InvalidProvider(error.to_string())),
+    };
+    let config = match provider.provider {
         LlmProviders::AwsTextract => match TextractOperation::from_model(provider.model)? {
             TextractOperation::DetectDocumentText => OcrConfigKind::AwsTextract,
             TextractOperation::AnalyzeDocument => OcrConfigKind::AwsTextractAnalyze,
@@ -227,11 +281,11 @@ pub(crate) fn resolve_provider_config(
         LlmProviders::VertexAi => OcrConfigKind::VertexAi,
         _ => {
             return Err(Error::InvalidProvider(
-                provider.custom_llm_provider.to_string(),
+                <&'static str>::from(provider.provider).to_string(),
             ));
         }
     };
-    Ok((provider.model.to_string(), config))
+    Ok((provider, config))
 }
 
 fn is_document_intelligence_model(model: &str) -> bool {
@@ -243,7 +297,7 @@ fn is_document_intelligence_model(model: &str) -> bool {
 mod tests {
     use std::collections::HashSet;
 
-    use litellm_auth::{InputSource, Sourced};
+    use litellm_auth::Sourced;
     use litellm_llms::{
         base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document,
     };
@@ -484,8 +538,8 @@ mod tests {
         #[case] expected_config: OcrConfigKind,
     ) {
         let expected_model = qualified_model.split_once('/').unwrap().1;
-        let (model, config) = resolve_provider_config(qualified_model, None).unwrap();
-        assert_eq!(model, expected_model);
+        let (provider, config) = resolve_provider_config(qualified_model, None).unwrap();
+        assert_eq!(provider.model, expected_model);
         assert_eq!(config, expected_config);
     }
 
@@ -528,7 +582,7 @@ mod tests {
             expected_config
         );
         assert_eq!(
-            resolve_provider_config(model, None).unwrap().0,
+            resolve_provider_config(model, None).unwrap().0.model,
             model.split_once('/').unwrap().1
         );
     }

litellm-rust/crates/inference-ocr/src/types.rs

@@ -75,6 +75,8 @@ impl OcrConnectionInputs {
 
 pub struct LiteLLMOcrRequest<D = OcrDocumentInput> {
     pub model: String,
+    pub(crate) requested_model: String,
+    pub(crate) custom_llm_provider: Option<String>,
     pub document: D,
     pub credentials: OcrCredentialInputs,
     pub transport: OcrTransportConfig,
@@ -91,7 +93,8 @@ impl LiteLLMOcrRequest {
         custom_llm_provider: Option<&str>,
         optional_params: CallArguments,
     ) -> Result<Self, Error> {
-        let (model, config) = resolve_provider_config(&model, custom_llm_provider)?;
+        let (provider, config) = resolve_provider_config(&model, custom_llm_provider)?;
+        let resolved_model = provider.model.to_owned();
         let default_transport = OcrTransportConfig::default();
         let max_response_bytes = optional_params
             .get("max_response_bytes")
@@ -116,7 +119,9 @@ impl LiteLLMOcrRequest {
             .collect();
 
         Ok(Self {
-            model,
+            model: resolved_model,
+            requested_model: model,
+            custom_llm_provider: custom_llm_provider.map(str::to_owned),
             document: document.into(),
             credentials: OcrCredentialInputs::default(),
             transport,
@@ -135,6 +140,8 @@ impl<D> LiteLLMOcrRequest<D> {
     ) -> Result<LiteLLMOcrRequest<T>, E> {
         Ok(LiteLLMOcrRequest {
             model: self.model,
+            requested_model: self.requested_model,
+            custom_llm_provider: self.custom_llm_provider,
             document: map(self.document)?,
             credentials: self.credentials,
             transport: self.transport,
@@ -148,6 +155,8 @@ impl<D> LiteLLMOcrRequest<D> {
     pub fn with_document<T>(self, document: T) -> LiteLLMOcrRequest<T> {
         LiteLLMOcrRequest {
             model: self.model,
+            requested_model: self.requested_model,
+            custom_llm_provider: self.custom_llm_provider,
             document,
             credentials: self.credentials,
             transport: self.transport,

litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs

@@ -66,6 +66,19 @@ async fn interception_runs_once_and_observers_receive_ordered_success_events() {
     assert_eq!(received(&upstream).await.len(), 1);
 }
 
+#[rstest::rstest]
+#[case::mutated_model("model-b")]
+#[tokio::test]
+async fn provider_request_uses_the_request_model_after_it_is_mutated(#[case] model: &str) {
+    let upstream = upstream([pages_response()]).await;
+    let mut request = ocr_request("mistral/model-a", &upstream.uri(), json!({}));
+    request.model = model.into();
+
+    perform(request).await.unwrap();
+
+    assert_eq!(only_request(&upstream).await.json()["model"], json!(model));
+}
+
 #[rstest::rstest]
 #[tokio::test]
 async fn a_blocking_before_send_prevents_the_call_and_emits_one_failure() {

litellm-rust/crates/inference-responses/AGENTS.md

@@ -2,3 +2,4 @@
 - This crate owns Responses API call orchestration. Payloads belong in `litellm-llms-types`, transformations in `llms/src/base_llm/responses` and `llms/src/<provider>/responses`
 - The HTTP route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks, cached through `litellm_inference::caching::execute_streaming`
 - WebSocket sessions (`src/websocket.rs`) stay separate from the HTTP call driver because a connection can accept multiple requests while receiving events
+- `src/ids.rs` owns the LiteLLM-managed ids handed back to callers (`resp_`, `encitem_`, `litellm_enc:`, `cntr_`) and must stay fully compatible with `litellm/responses/utils.py`

litellm-rust/crates/inference-responses/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+base64.workspace = true
 bytes.workspace = true
 futures-util.workspace = true
 litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }

litellm-rust/crates/inference-responses/src/handler.rs

@@ -3,39 +3,61 @@ use litellm_host::observation::ObservationSender;
 use std::time::Duration;
 
 use futures_util::StreamExt;
-use litellm_host::interceptors::{Interceptors, RawResponse, WireRequest};
-use litellm_llms::base_llm::auth::{Authenticated, resolve_auth};
+use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
+use litellm_llms::base_llm::auth::{AuthScheme, Authenticated, resolve_auth};
+use litellm_secrets::source::Secrets;
 
 use super::{
-    Error,
-    types::{ProviderResponsesRequest, ResponsesOutput, ResponsesStreamHead},
+    Error, ResponsesRoute,
+    types::{ResponsesCall, ResponsesOutput, ResponsesProvider, ResponsesStreamHead},
 };
 
 pub(super) async fn execute(
-    http: &litellm_http::Client,
-    auth: &litellm_auth::AuthServices,
-    request: ProviderResponsesRequest,
-    cache: Option<litellm_cache_response::ScopedCache>,
+    route: &ResponsesRoute,
+    ResponsesProvider { identity, config }: ResponsesProvider,
+    call: ResponsesCall,
+    secrets: Secrets,
     cache_options: Option<litellm_cache_response::CachePolicy>,
     interceptors: &impl Interceptors<Error>,
     observers: Option<&ObservationSender>,
 ) -> Result<ResponsesOutput, Error> {
-    let authenticated = resolve_auth(auth, request.environment, &|_| None).await?;
-    let identity = litellm_host::interceptors::ProviderIdentity {
-        model: request.context.model.clone(),
-        provider: request.context.custom_llm_provider.clone(),
+    let lookup = |name: &str| secrets.get(name);
+    let environment = config.validate_environment(
+        litellm_http::request::string_headers("responses", call.extra_headers)?,
+        call.api_key.as_deref(),
+        &lookup,
+    )?;
+    let url = config.get_complete_url(call.api_base.as_deref(), &lookup);
+    let context = RequestContext {
+        model: identity.model.clone(),
+        custom_llm_provider: identity.provider.clone(),
+        optional_params: serde_json::Value::Object(call.optional_params.clone()),
+        secret_fields: Vec::new(),
+        api_key: match &environment.auth {
+            AuthScheme::Credential { secret, .. } => Some(secret.clone()),
+            _ => None,
+        },
     };
+    let body = config.transform_responses_api_request(
+        &identity.model,
+        call.input,
+        call.optional_params,
+    )?;
+    let authenticated = resolve_auth(&route.auth, environment, &|_| None).await?;
     let wire = interceptors
         .before_provider_request(
             WireRequest {
-                url: request.url,
+                url,
                 headers: authenticated.headers,
-                body: request.body,
+                body,
             },
-            request.context,
+            context,
         )
         .await?;
-    let cache = cache.filter(|_| authenticated.signer.is_none());
+    let cache = route
+        .cache
+        .clone()
+        .filter(|_| authenticated.signer.is_none());
     let cache_request = litellm_inference::caching::CacheRequest::from_wire(
         identity,
         cache.as_ref().map(|_| &wire),
@@ -59,9 +81,9 @@ pub(super) async fn execute(
                 },
                 wire.url,
                 &wire.body,
-                Some(request.timeout.unwrap_or(Duration::from_secs(600))),
+                Some(call.timeout.unwrap_or(Duration::from_secs(600))),
             )?;
-            let response = litellm_inference::outbound::send(outbound, http)
+            let response = litellm_inference::outbound::send(outbound, &route.http)
                 .await
                 .map_err(network)?;
             let status = response.status().as_u16();
@@ -103,8 +125,7 @@ pub(super) async fn execute(
                 .map_err(Error::post_call)?;
             let value = serde_json::from_str(&body)
                 .map_err(|error| Error::InvalidResponse(error.to_string().into()))?;
-            request
-                .config
+            config
                 .transform_response_api_response(value)
                 .map(ResponsesOutput::Complete)
                 .map_err(Error::from)

litellm-rust/crates/inference-responses/src/ids.rs

@@ -0,0 +1,499 @@
+//! LiteLLM-managed identifiers on Responses payloads, fully compatible with `litellm/responses/utils.py`
+//!
+//! LiteLLM rewrites some ids in a Responses API result before handing it back, so that a later
+//! request carrying those ids can be routed to the provider and deployment that produced them,
+//! and then restored to what the provider originally issued. The native and Python paths serve
+//! the same traffic, so an id minted by either one must decode on the other.
+//!
+//! Each managed id is a fixed prefix followed by standard, padded base64 of a `;`-separated
+//! plaintext:
+//!
+//! | Id | Applied to | Decoded plaintext |
+//! |---|---|---|
+//! | `resp_<b64>` | the response `id` | `litellm:custom_llm_provider:{provider};model_id:{model_id};response_id:{id}` |
+//! | `encitem_<b64>` | the `id` of an output item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` |
+//! | `litellm_enc:<b64>;{content}` | that item's `encrypted_content` | `<b64>` is `model_id:{model_id}`, and the original content follows the `;` unencoded |
+//! | `cntr_<b64>` | `container_id` on output items, `code_interpreter_call`, and message annotations | `litellm:custom_llm_provider:{provider};model_id:{model_id};container_id:{id}` |
+//!
+//! For example, OpenAI's `resp_0a1b2c3d4e5f` served without a deployment id becomes
+//! `resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm`,
+//! which decodes to `litellm:custom_llm_provider:openai;model_id:None;response_id:resp_0a1b2c3d4e5f`.
+//!
+//! How they fit together:
+//! - `resp_` lets a follow-up's `previous_response_id` find the provider and deployment again.
+//!   [`encode_response`] stamps it on every response, unless the id is already managed.
+//! - `encitem_` and `litellm_enc:` carry encrypted-content affinity: encrypted reasoning only
+//!   decrypts under the API key that produced it, so the router pins replays to that deployment.
+//!   They are stamped only when the router sets `encrypted_content_affinity_enabled` and a
+//!   deployment `model_id` is known (see [`IdScope`]). Items without an `id`, such as Codex's,
+//!   carry the deployment in the wrapped content alone.
+//! - `cntr_` lets container calls find the provider and deployment that own a container. A
+//!   container id that already names a provider is left as it is.
+//! - On the way in, [`restore_input`] turns `encitem_` ids and `litellm_enc:` content in the
+//!   request input back into the provider's own bytes before the request is sent. The routing
+//!   decision itself stays in Python's `EncryptedContentAffinityCheck`.
+//!
+//! Python's quirks are kept on purpose:
+//! - A missing provider or `model_id` is written as `None` in `resp_` ids (an f-string of
+//!   `None`), but as an empty string in `cntr_` ids. [`decode_container_id`] reads both as absent.
+//! - Decoding follows Python's non-strict `base64.b64decode`: characters outside the alphabet
+//!   are skipped, trailing bits are allowed, and padding stripped in transit is added back.
+//! - A `resp_` id counts as managed when it decodes to at least three `;`-separated parts.
+
+use base64::{
+    Engine, alphabet,
+    engine::{GeneralPurpose, general_purpose::PAD},
+};
+use litellm_llms_types::formats::responses::ResponsesApiResponse;
+use serde_json::{Map, Value};
+
+const RESPONSE_PREFIX: &str = "resp_";
+const ITEM_PREFIX: &str = "encitem_";
+const CONTAINER_PREFIX: &str = "cntr_";
+const WRAPPED_CONTENT_PREFIX: &str = "litellm_enc:";
+
+/// Python's non-strict `base64.b64decode`: characters outside the alphabet are skipped and
+/// non-zero trailing bits are accepted
+const LENIENT: GeneralPurpose = GeneralPurpose::new(
+    &alphabet::STANDARD,
+    PAD.with_decode_allow_trailing_bits(true),
+);
+
+/// Where a response came from, as encoded into the ids LiteLLM hands back
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct IdScope {
+    pub provider: Option<String>,
+    pub model_id: Option<String>,
+    pub encrypted_content_affinity: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DecodedContainerId {
+    pub provider: Option<String>,
+    pub model_id: Option<String>,
+    pub container_id: String,
+}
+
+/// Rewrites the response id, encrypted-content items and container ids the way the Python
+/// Responses path does before returning to the caller. An already managed id is left untouched
+pub fn encode_response(response: &mut ResponsesApiResponse, scope: &IdScope) {
+    if is_managed_response_id(&response.id) {
+        return;
+    }
+    response.id = response_id(scope, &response.id);
+    if let Some(model_id) = scope
+        .model_id
+        .as_deref()
+        .filter(|_| scope.encrypted_content_affinity)
+        .filter(|model_id| !model_id.is_empty())
+    {
+        response
+            .output
+            .iter_mut()
+            .filter_map(Value::as_object_mut)
+            .for_each(|item| encode_encrypted_item(item, model_id));
+    }
+    response
+        .output
+        .iter_mut()
+        .for_each(|item| encode_containers(item, scope));
+}
+
+/// Restores encrypted-content item ids and contents that LiteLLM encoded on an earlier response,
+/// so the provider receives what it originally issued
+pub fn restore_input(input: &mut Value) {
+    let Some(items) = input.as_array_mut() else {
+        return;
+    };
+    for item in items.iter_mut().filter_map(Value::as_object_mut) {
+        if let Some(Value::String(id)) = item.get_mut("id")
+            && let Some((_, original)) = decode_encrypted_item_id(id)
+        {
+            *id = original;
+        }
+        if let Some(Value::String(content)) = item.get_mut("encrypted_content")
+            && let Some(original) = unwrap_encrypted_content(content)
+        {
+            *content = original;
+        }
+    }
+}
+
+pub fn response_id(scope: &IdScope, id: &str) -> String {
+    let assembled = format!(
+        "litellm:custom_llm_provider:{};model_id:{};response_id:{id}",
+        python_str(scope.provider.as_deref()),
+        python_str(scope.model_id.as_deref()),
+    );
+    format!("{RESPONSE_PREFIX}{}", LENIENT.encode(assembled))
+}
+
+pub fn is_managed_response_id(id: &str) -> bool {
+    decode_text(&id.replace(RESPONSE_PREFIX, ""))
+        .is_some_and(|decoded| decoded.split(';').count() >= 3)
+}
+
+pub fn encrypted_item_id(model_id: &str, id: &str) -> String {
+    format!(
+        "{ITEM_PREFIX}{}",
+        LENIENT.encode(format!("litellm:model_id:{model_id};item_id:{id}"))
+    )
+}
+
+pub fn decode_encrypted_item_id(id: &str) -> Option<(String, String)> {
+    let decoded = decode_text(&padded(id.strip_prefix(ITEM_PREFIX)?))?;
+    let (model_id, item_id) = decoded.split_once(';')?;
+    Some((
+        model_id.replace("litellm:model_id:", ""),
+        item_id.replace("item_id:", ""),
+    ))
+}
+
+pub fn wrap_encrypted_content(content: &str, model_id: &str) -> String {
+    format!(
+        "{WRAPPED_CONTENT_PREFIX}{};{content}",
+        LENIENT.encode(format!("model_id:{model_id}"))
+    )
+}
+
+/// The original content of a wrapped `encrypted_content`, or `None` when it is not wrapped
+pub fn unwrap_encrypted_content(content: &str) -> Option<String> {
+    if !content.starts_with(WRAPPED_CONTENT_PREFIX) {
+        return None;
+    }
+    let (metadata, original) = content.split_once(';')?;
+    decode_text(&padded(&metadata.replace(WRAPPED_CONTENT_PREFIX, "")))?;
+    Some(original.to_owned())
+}
+
+pub fn container_id(scope: &IdScope, id: &str) -> String {
+    let assembled = format!(
+        "litellm:custom_llm_provider:{};model_id:{};container_id:{id}",
+        scope.provider.as_deref().unwrap_or_default(),
+        scope.model_id.as_deref().unwrap_or_default(),
+    );
+    format!("{CONTAINER_PREFIX}{}", LENIENT.encode(assembled))
+}
+
+pub fn decode_container_id(id: &str) -> DecodedContainerId {
+    let unmanaged = || DecodedContainerId {
+        provider: None,
+        model_id: None,
+        container_id: id.to_owned(),
+    };
+    if !id.starts_with(CONTAINER_PREFIX) {
+        return unmanaged();
+    }
+    let Some(decoded) = decode_text(&id.replace(CONTAINER_PREFIX, "")) else {
+        return unmanaged();
+    };
+    let parsed = decoded
+        .strip_prefix("litellm:custom_llm_provider:")
+        .and_then(|rest| rest.split_once(';'))
+        .and_then(|(provider, rest)| {
+            let (model_id, rest) = rest.strip_prefix("model_id:")?.split_once(';')?;
+            let container_id = rest.strip_prefix("container_id:")?;
+            (!container_id.is_empty() && !container_id.contains('\n')).then_some((
+                provider,
+                model_id,
+                container_id,
+            ))
+        });
+    let Some((provider, model_id, container_id)) = parsed else {
+        return unmanaged();
+    };
+    let present = |value: &str| (!matches!(value, "" | "None")).then(|| value.to_owned());
+    DecodedContainerId {
+        provider: present(provider),
+        model_id: present(model_id),
+        container_id: container_id.to_owned(),
+    }
+}
+
+fn encode_encrypted_item(item: &mut Map<String, Value>, model_id: &str) {
+    let Some(Value::String(content)) = item.get_mut("encrypted_content") else {
+        return;
+    };
+    if content.is_empty() {
+        return;
+    }
+    *content = wrap_encrypted_content(content, model_id);
+    if let Some(Value::String(id)) = item.get_mut("id")
+        && !id.is_empty()
+    {
+        *id = encrypted_item_id(model_id, id);
+    }
+}
+
+fn encode_containers(item: &mut Value, scope: &IdScope) {
+    let Some(item) = item.as_object_mut() else {
+        return;
+    };
+    encode_container_field(item, scope);
+    if let Some(nested) = item
+        .get_mut("code_interpreter_call")
+        .and_then(Value::as_object_mut)
+    {
+        encode_container_field(nested, scope);
+    }
+    if item.get("type").and_then(Value::as_str) != Some("message") {
+        return;
+    }
+    let Some(content) = item.get_mut("content").and_then(Value::as_array_mut) else {
+        return;
+    };
+    content
+        .iter_mut()
+        .filter_map(|part| part.get_mut("annotations").and_then(Value::as_array_mut))
+        .flatten()
+        .for_each(|annotation| encode_containers(annotation, scope));
+}
+
+fn encode_container_field(item: &mut Map<String, Value>, scope: &IdScope) {
+    if let Some(Value::String(id)) = item.get_mut("container_id")
+        && decode_container_id(id).provider.is_none()
+    {
+        *id = container_id(scope, id);
+    }
+}
+
+fn python_str(value: Option<&str>) -> &str {
+    value.unwrap_or("None")
+}
+
+fn padded(value: &str) -> String {
+    let missing = value.len() % 4;
+    let padding = if missing == 0 { 0 } else { 4 - missing };
+    format!("{value}{}", "=".repeat(padding))
+}
+
+fn decode_text(value: &str) -> Option<String> {
+    let alphabet: String = value
+        .chars()
+        .filter(|character| {
+            character.is_ascii_alphanumeric() || matches!(character, '+' | '/' | '=')
+        })
+        .collect();
+    String::from_utf8(LENIENT.decode(alphabet).ok()?).ok()
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    fn scope(provider: Option<&str>, model_id: Option<&str>, affinity: bool) -> IdScope {
+        IdScope {
+            provider: provider.map(str::to_owned),
+            model_id: model_id.map(str::to_owned),
+            encrypted_content_affinity: affinity,
+        }
+    }
+
+    fn response(id: &str, output: Value) -> ResponsesApiResponse {
+        serde_json::from_value(json!({"id": id, "model": "gpt-6.1-sol", "output": output})).unwrap()
+    }
+
+    #[rstest]
+    #[case::openai_without_model_id(
+        Some("openai"),
+        None,
+        "resp_0a1b2c3d4e5f",
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm"
+    )]
+    #[case::openrouter_with_model_id(
+        Some("openrouter"),
+        Some("deploy-1"),
+        "gen-123",
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5yb3V0ZXI7bW9kZWxfaWQ6ZGVwbG95LTE7cmVzcG9uc2VfaWQ6Z2VuLTEyMw=="
+    )]
+    fn response_ids_match_python(
+        #[case] provider: Option<&str>,
+        #[case] model_id: Option<&str>,
+        #[case] id: &str,
+        #[case] expected: &str,
+    ) {
+        assert_eq!(response_id(&scope(provider, model_id, false), id), expected);
+    }
+
+    #[rstest]
+    #[case::raw_short_id("resp_0a1b2c3d4e5f", false)]
+    #[case::raw_long_id("resp_68a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3", false)]
+    #[case::two_segments("resp_YTti", false)]
+    #[case::three_segments("resp_YTtiO2M=", true)]
+    #[case::managed(
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOng=",
+        true
+    )]
+    fn managed_response_ids_match_python(#[case] id: &str, #[case] expected: bool) {
+        assert_eq!(is_managed_response_id(id), expected);
+    }
+
+    #[test]
+    fn encrypted_items_match_python() {
+        assert_eq!(
+            encrypted_item_id("deploy-1", "rs_0a1b"),
+            "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI="
+        );
+        assert_eq!(
+            wrap_encrypted_content("gAAAA", "deploy-1"),
+            "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA"
+        );
+    }
+
+    #[rstest]
+    #[case::padded("encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", Some(("deploy-1", "rs_0a1b")))]
+    #[case::padding_stripped_in_transit("encitem_bGl0ZWxsbTptb2RlbF9pZDptO2l0ZW1faWQ6cnNfMQ", Some(("m", "rs_1")))]
+    #[case::not_base64("encitem_notbase64!", None)]
+    #[case::unmanaged("rs_0a1b", None)]
+    fn encrypted_item_ids_decode_like_python(
+        #[case] id: &str,
+        #[case] expected: Option<(&str, &str)>,
+    ) {
+        assert_eq!(
+            decode_encrypted_item_id(id),
+            expected.map(|(model_id, item_id)| (model_id.to_owned(), item_id.to_owned()))
+        );
+    }
+
+    #[rstest]
+    #[case::unpadded_metadata("litellm_enc:bW9kZWxfaWQ6bQ;gAAAA", Some("gAAAA"))]
+    #[case::no_separator("litellm_enc:nosemicolon", None)]
+    #[case::unwrapped("gAAAA", None)]
+    fn encrypted_content_unwraps_like_python(
+        #[case] content: &str,
+        #[case] expected: Option<&str>,
+    ) {
+        assert_eq!(unwrap_encrypted_content(content).as_deref(), expected);
+    }
+
+    #[rstest]
+    #[case::with_model_id(
+        Some("deploy-1"),
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpkZXBsb3ktMTtjb250YWluZXJfaWQ6Y250cl9hYmM="
+    )]
+    #[case::without_model_id(
+        None,
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfYWJj"
+    )]
+    fn container_ids_match_python(#[case] model_id: Option<&str>, #[case] expected: &str) {
+        assert_eq!(
+            container_id(&scope(Some("openai"), model_id, false), "cntr_abc"),
+            expected
+        );
+    }
+
+    #[rstest]
+    #[case::raw("cntr_abc", None, None, "cntr_abc")]
+    #[case::managed(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDptO2NvbnRhaW5lcl9pZDpjbnRyXzE=",
+        Some("openai"),
+        Some("m"),
+        "cntr_1"
+    )]
+    #[case::empty_segments(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOjttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfMQ==",
+        None,
+        None,
+        "cntr_1"
+    )]
+    #[case::python_none_provider(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOk5vbmU7bW9kZWxfaWQ6O2NvbnRhaW5lcl9pZDp4",
+        None,
+        None,
+        "x"
+    )]
+    fn container_ids_decode_like_python(
+        #[case] id: &str,
+        #[case] provider: Option<&str>,
+        #[case] model_id: Option<&str>,
+        #[case] container_id: &str,
+    ) {
+        assert_eq!(
+            decode_container_id(id),
+            DecodedContainerId {
+                provider: provider.map(str::to_owned),
+                model_id: model_id.map(str::to_owned),
+                container_id: container_id.to_owned(),
+            }
+        );
+    }
+
+    #[test]
+    fn encodes_response_items_under_affinity() {
+        let mut encoded = response(
+            "resp_1",
+            json!([
+                {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
+                {"type": "reasoning", "encrypted_content": "gBBBB", "summary": []},
+                {"type": "code_interpreter_call", "id": "ci_1", "container_id": "cntr_abc"},
+                {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
+                    {"type": "container_file_citation", "container_id": "cntr_abc"}
+                ]}]}
+            ]),
+        );
+        let scope = scope(Some("openai"), Some("deploy-1"), true);
+        encode_response(&mut encoded, &scope);
+        let container = container_id(&scope, "cntr_abc");
+        assert_eq!(encoded.id, response_id(&scope, "resp_1"));
+        assert_eq!(
+            Value::Array(encoded.output),
+            json!([
+                {
+                    "type": "reasoning",
+                    "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
+                    "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
+                    "summary": []
+                },
+                {"type": "reasoning", "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gBBBB", "summary": []},
+                {"type": "code_interpreter_call", "id": "ci_1", "container_id": container},
+                {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
+                    {"type": "container_file_citation", "container_id": container}
+                ]}]}
+            ])
+        );
+    }
+
+    #[rstest]
+    #[case::affinity_disabled(scope(Some("openai"), Some("deploy-1"), false))]
+    #[case::no_model_id(scope(Some("openai"), None, true))]
+    fn leaves_encrypted_items_without_affinity(#[case] scope: IdScope) {
+        let output = json!([{"type": "reasoning", "id": "rs_1", "encrypted_content": "gAAAA", "summary": []}]);
+        let mut encoded = response("resp_1", output.clone());
+        encode_response(&mut encoded, &scope);
+        assert_eq!(Value::Array(encoded.output), output);
+    }
+
+    #[test]
+    fn leaves_managed_responses_untouched() {
+        let scope = scope(Some("openai"), Some("deploy-1"), true);
+        let id = response_id(&scope, "resp_1");
+        let output = json!([{"type": "code_interpreter_call", "container_id": "cntr_abc"}]);
+        let mut encoded = response(&id, output.clone());
+        encode_response(&mut encoded, &scope);
+        assert_eq!((encoded.id, Value::Array(encoded.output)), (id, output));
+    }
+
+    #[test]
+    fn restores_encrypted_items_in_input() {
+        let mut input = json!([
+            {
+                "type": "reasoning",
+                "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
+                "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
+                "summary": []
+            },
+            {"role": "user", "content": "hi"}
+        ]);
+        restore_input(&mut input);
+        assert_eq!(
+            input,
+            json!([
+                {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
+                {"role": "user", "content": "hi"}
+            ])
+        );
+    }
+}

litellm-rust/crates/inference-responses/src/lib.rs

@@ -1,19 +1,36 @@
 use litellm_host::observation::ObservationSender;
 pub use litellm_inference::RouteError as Error;
 
+pub mod ids;
 pub mod route;
 pub mod types;
 pub mod websocket;
 
 mod handler;
-mod prepare;
 
 use std::sync::Arc;
 
 use litellm_auth::AuthServices;
+use litellm_core_utils::get_llm_provider_logic::{
+    CustomLlmProvider, LlmProviders, get_custom_llm_provider,
+};
 use litellm_host::interceptors::Interceptors;
+use litellm_host::interceptors::ProviderIdentity;
+use litellm_llms::{
+    base_llm::responses::transformation::BaseResponsesApiConfig,
+    openai::responses::transformation::OpenAiResponsesApiConfig,
+    openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
+};
 use litellm_secrets::source::SecretSource;
-use types::{ResponsesCall, ResponsesOutput};
+use types::{ResponsesCall, ResponsesOutput, ResponsesProvider};
+
+fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> {
+    match provider {
+        LlmProviders::Openai => Some(&OpenAiResponsesApiConfig),
+        LlmProviders::Openrouter => Some(&OPENROUTER_RESPONSES_CONFIG),
+        _ => None,
+    }
+}
 
 #[derive(Clone)]
 pub struct ResponsesRoute {
@@ -77,34 +94,68 @@ impl ResponsesRoute {
         observers: Option<&ObservationSender>,
     ) -> Result<ResponsesOutput, Error> {
         litellm_inference::diagnostic::call(async {
-            self.run_provider(call, cache_options, interceptors, observers)
-                .await
+            let explicit = call
+                .custom_llm_provider
+                .as_deref()
+                .filter(|provider| !provider.is_empty());
+            let (provider_name, resolved_model) =
+                match get_custom_llm_provider(&call.model, explicit) {
+                    Some(CustomLlmProvider {
+                        model,
+                        custom_llm_provider,
+                    }) => (custom_llm_provider, model),
+                    None => ("openai", call.model.as_str()),
+                };
+            if resolved_model.is_empty() {
+                return Err(Error::InvalidProvider(call.model));
+            }
+            let provider = provider_name
+                .parse::<LlmProviders>()
+                .map_err(|_| Error::Unsupported("native HTTP responses provider"))?;
+            let config = responses_provider(provider)
+                .ok_or(Error::Unsupported("native HTTP responses provider"))?;
+            let wire_model = match provider {
+                LlmProviders::Openai if resolved_model.contains('/') => {
+                    return Err(Error::InvalidProvider(call.model));
+                }
+                LlmProviders::Openai => resolved_model.into(),
+                LlmProviders::Openrouter if explicit.is_some() && !resolved_model.contains('/') => {
+                    format!("openrouter/{resolved_model}")
+                }
+                LlmProviders::Openrouter => resolved_model.into(),
+                _ => return Err(Error::Unsupported("native HTTP responses provider")),
+            };
+            let provider = ResponsesProvider {
+                identity: ProviderIdentity {
+                    model: wire_model,
+                    provider: provider_name.into(),
+                },
+                config,
+            };
+            litellm_inference::diagnostic::provider(
+                &provider.identity.model,
+                &provider.identity.provider,
+            );
+            let secrets = self
+                .secrets
+                .resolve(
+                    provider
+                        .config
+                        .secret_names(call.api_key.as_deref(), call.api_base.as_deref()),
+                )
+                .await?;
+            let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
+                Box::pin(handler::execute(
+                    self,
+                    provider,
+                    call,
+                    secrets,
+                    cache_options,
+                    interceptors,
+                    observers,
+                ));
+            execute.await
         })
         .await
     }
-
-    async fn run_provider(
-        &self,
-        call: ResponsesCall,
-        cache_options: Option<litellm_cache_response::CachePolicy>,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<ResponsesOutput, Error> {
-        let request = prepare::prepare(call, self.secrets.as_ref()).await?;
-        litellm_inference::diagnostic::provider(
-            &request.context.model,
-            &request.context.custom_llm_provider,
-        );
-        let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
-            Box::pin(handler::execute(
-                &self.http,
-                &self.auth,
-                request,
-                self.cache.clone(),
-                cache_options,
-                interceptors,
-                observers,
-            ));
-        execute.await
-    }
 }

litellm-rust/crates/inference-responses/src/prepare.rs

@@ -1,162 +0,0 @@
-use litellm_core_utils::get_llm_provider_logic::{
-    CustomLlmProvider, LlmProviders, get_custom_llm_provider,
-};
-use litellm_host::interceptors::{ProviderIdentity, RequestContext};
-use litellm_llms::{
-    base_llm::responses::transformation::BaseResponsesApiConfig,
-    openai::responses::transformation::OpenAiResponsesApiConfig,
-    openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
-};
-use litellm_secrets::source::SecretSource;
-use serde_json::Value;
-
-use super::{
-    Error,
-    types::{ProviderResponsesRequest, ResponsesCall},
-};
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) async fn prepare(
-    call: ResponsesCall,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderResponsesRequest, Error> {
-    let ResponsesProvider { identity, config } =
-        resolve_provider(&call.model, call.custom_llm_provider.as_deref())?;
-    let provider = identity.provider.as_str();
-    let model = identity.model.as_str();
-    let snapshot = secrets
-        .resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref()))
-        .await?;
-    let lookup = |name: &str| snapshot.get(name);
-    let environment = config.validate_environment(
-        litellm_http::request::string_headers("responses", call.extra_headers)?,
-        call.api_key.as_deref(),
-        &lookup,
-    )?;
-    let context = RequestContext {
-        model: model.into(),
-        custom_llm_provider: provider.into(),
-        optional_params: Value::Object(call.optional_params.clone()),
-        secret_fields: Vec::new(),
-        api_key: match &environment.auth {
-            litellm_llms::base_llm::auth::AuthScheme::Credential { secret, .. } => {
-                Some(secret.clone())
-            }
-            _ => None,
-        },
-    };
-    let body = config.transform_responses_api_request(model, call.input, call.optional_params)?;
-    Ok(ProviderResponsesRequest {
-        url: config.get_complete_url(call.api_base.as_deref(), &lookup),
-        config,
-        environment,
-        body,
-        context,
-        timeout: call.timeout,
-    })
-}
-
-pub(super) struct ResponsesProvider {
-    pub identity: ProviderIdentity,
-    pub config: &'static dyn BaseResponsesApiConfig,
-}
-
-/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter
-/// provider a model without a vendor segment keeps the `openrouter/` prefix as its wire id.
-pub(super) fn resolve_provider(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<ResponsesProvider, Error> {
-    let explicit = custom_llm_provider.filter(|provider| !provider.is_empty());
-    let (provider, resolved) = match get_custom_llm_provider(model, explicit) {
-        Some(CustomLlmProvider {
-            model,
-            custom_llm_provider,
-        }) => (custom_llm_provider, model),
-        None => ("openai", model),
-    };
-    if resolved.is_empty() {
-        return Err(Error::InvalidProvider(model.into()));
-    }
-    let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) =
-        match provider.parse::<LlmProviders>() {
-            Ok(LlmProviders::Openai) if resolved.contains('/') => {
-                return Err(Error::InvalidProvider(model.into()));
-            }
-            Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()),
-            Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => (
-                &OPENROUTER_RESPONSES_CONFIG,
-                format!("openrouter/{resolved}"),
-            ),
-            Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()),
-            _ => return Err(Error::Unsupported("native HTTP responses provider")),
-        };
-    Ok(ResponsesProvider {
-        identity: ProviderIdentity {
-            model: wire_model,
-            provider: provider.into(),
-        },
-        config,
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use rstest::rstest;
-
-    use super::*;
-
-    /// Pinned against Python's `get_llm_provider` on 2026-10-11.
-    #[rstest]
-    #[case::bare("gpt-5", None, "openai", "gpt-5")]
-    #[case::openai_prefix("openai/gpt-5", None, "openai", "gpt-5")]
-    #[case::openai_explicit("gpt-5", Some("openai"), "openai", "gpt-5")]
-    #[case::openrouter_prefix(
-        "openrouter/anthropic/claude",
-        None,
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_explicit(
-        "openrouter/anthropic/claude",
-        Some("openrouter"),
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_explicit_unprefixed(
-        "anthropic/claude",
-        Some("openrouter"),
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_native_id(
-        "openrouter/auto",
-        Some("openrouter"),
-        "openrouter",
-        "openrouter/auto"
-    )]
-    #[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")]
-    fn providers_and_wire_models_match_python(
-        #[case] model: &str,
-        #[case] custom_llm_provider: Option<&str>,
-        #[case] provider: &str,
-        #[case] wire_model: &str,
-    ) {
-        let resolved = resolve_provider(model, custom_llm_provider).unwrap();
-        assert_eq!(resolved.identity.provider, provider);
-        assert_eq!(resolved.identity.model, wire_model);
-    }
-
-    #[rstest]
-    #[case::other_prefix("other/model", None)]
-    #[case::other_explicit("model", Some("other"))]
-    #[case::openai_with_vendor_segment("openai/other/model", None)]
-    #[case::empty_model("openai/", None)]
-    #[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
-    fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) {
-        assert!(matches!(
-            resolve_provider(model, custom_llm_provider),
-            Err(Error::InvalidProvider(_) | Error::Unsupported(_))
-        ));
-    }
-}

litellm-rust/crates/inference-responses/src/types.rs

@@ -2,9 +2,8 @@ use std::time::Duration;
 
 use bytes::Bytes;
 use litellm_host::call::CallOutput;
-use litellm_llms::base_llm::{
-    auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig,
-};
+use litellm_host::interceptors::ProviderIdentity;
+use litellm_llms::base_llm::responses::transformation::BaseResponsesApiConfig;
 use litellm_llms_types::formats::responses::ResponsesApiResponse;
 use serde_json::{Map, Value};
 
@@ -21,17 +20,13 @@ pub struct ResponsesCall {
     pub timeout: Option<Duration>,
 }
 
+pub(crate) struct ResponsesProvider {
+    pub identity: ProviderIdentity,
+    pub config: &'static dyn BaseResponsesApiConfig,
+}
+
 pub struct ResponsesStreamHead {
     pub headers: Vec<(String, String)>,
 }
 
 pub type ResponsesOutput = CallOutput<ResponsesApiResponse, ResponsesStreamHead, Bytes, Error>;
-
-pub(super) struct ProviderResponsesRequest {
-    pub config: &'static dyn BaseResponsesApiConfig,
-    pub environment: ValidatedEnvironment,
-    pub url: String,
-    pub body: Value,
-    pub context: litellm_host::interceptors::RequestContext,
-    pub timeout: Option<Duration>,
-}

litellm-rust/crates/inference-responses/tests/responses.rs

@@ -7,6 +7,7 @@ use std::sync::Arc;
 use futures_util::TryStreamExt;
 use litellm_host::{call::HostedCompletion, lifecycle::CallEvent};
 use litellm_inference_responses::{
+    Error,
     route::Responses,
     types::{ResponsesCall, ResponsesOutput},
 };
@@ -32,6 +33,110 @@ fn call() -> ResponsesCall {
     }
 }
 
+#[rstest]
+#[case::bare_openai("gpt-5", None, "gpt-5")]
+#[case::openai_prefix("openai/gpt-5", None, "gpt-5")]
+#[case::empty_explicit_provider("openai/gpt-5", Some(""), "gpt-5")]
+#[case::explicit_openai("gpt-5", Some("openai"), "gpt-5")]
+#[case::openrouter_prefix("openrouter/anthropic/claude", None, "anthropic/claude")]
+#[case::explicit_openrouter_prefix(
+    "openrouter/anthropic/claude",
+    Some("openrouter"),
+    "anthropic/claude"
+)]
+#[case::explicit_openrouter_unprefixed("anthropic/claude", Some("openrouter"), "anthropic/claude")]
+#[case::explicit_openrouter_native_id("auto", Some("openrouter"), "openrouter/auto")]
+#[case::implicit_openrouter_native_id("openrouter/auto", None, "auto")]
+#[tokio::test]
+async fn provider_resolution_preserves_the_wire_model(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+    #[case] expected_wire_model: &str,
+) {
+    let response = json!({
+        "id": "response-1",
+        "model": expected_wire_model,
+        "output": [{"type": "message", "content": []}],
+        "usage": {"total_tokens": 7}
+    });
+    let upstream = upstream([json_response(response)]).await;
+    responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_key: Some("test-key".into()),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(
+        only_request(&upstream).await.json()["model"],
+        expected_wire_model
+    );
+}
+
+#[rstest]
+#[case::explicit_unknown_provider("test-model", Some("unknown-provider"))]
+#[case::unsupported_responses_provider("test-model", Some("anthropic"))]
+#[tokio::test]
+async fn unsupported_providers_fail_before_transport(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+) {
+    let upstream = upstream([]).await;
+    let result = responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await;
+
+    assert!(matches!(
+        result,
+        Err(Error::Unsupported("native HTTP responses provider"))
+    ));
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[case::openai_with_vendor_segment("openai/other/model", None)]
+#[case::empty_model("openai/", None)]
+#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
+#[tokio::test]
+async fn invalid_provider_models_fail_before_transport(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+) {
+    let upstream = upstream([]).await;
+    let result = responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await;
+
+    assert!(matches!(result, Err(Error::InvalidProvider(_))));
+    assert!(received(&upstream).await.is_empty());
+}
+
 #[rstest]
 #[case::direct(false)]
 #[case::hosted(true)]

litellm-rust/crates/inference-transcription/src/handler.rs

@@ -1,28 +1,57 @@
 use std::time::Duration;
 
-use litellm_http::{Client, request::truncate_error_body};
-use litellm_llms::base_llm::auth::resolve_auth;
+use litellm_http::{
+    Client,
+    request::{string_headers, truncate_error_body, with_default_headers},
+};
+use litellm_llms::base_llm::{
+    audio_transcription::transformation::BaseAudioTranscriptionConfig,
+    auth::{ValidatedEnvironment, resolve_auth},
+};
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
 use super::Error;
-use crate::{
-    constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::ProviderAudioTranscriptionRequest,
-};
+use crate::{constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::AudioTranscriptionRequest};
+use litellm_inference::provider::ResolvedProvider;
 
-pub async fn execute_audio_transcription_provider_call(
+pub(super) async fn execute(
     http: &Client,
     auth: &litellm_auth::AuthServices,
-    request: ProviderAudioTranscriptionRequest,
+    config: &'static dyn BaseAudioTranscriptionConfig,
+    provider: ResolvedProvider<'_>,
+    call: AudioTranscriptionRequest<'_>,
+    secrets: Secrets,
 ) -> Result<Value, Error> {
-    let env_lookup = |key: &str| request.secrets.get(key);
-    let authenticated = resolve_auth(auth, request.environment.clone(), &env_lookup).await?;
+    let env_lookup = |key: &str| secrets.get(key);
+    let forwarded = string_headers("audio transcription", call.extra_headers)?;
+    let validated = config.validate_environment(
+        forwarded,
+        provider.model,
+        &call.optional_params,
+        &env_lookup,
+    )?;
+    let environment = ValidatedEnvironment {
+        headers: with_default_headers(validated.headers, config.default_headers()),
+        auth: validated.auth,
+    };
+    let url = config.get_complete_url(
+        call.api_base,
+        provider.model,
+        &call.optional_params,
+        &env_lookup,
+    )?;
+    let filtered_params = config.map_transcription_params(&call.optional_params);
+    let body = config
+        .transform_audio_transcription_request(provider.model, call.audio, filtered_params)?
+        .body;
+    let authenticated = resolve_auth(auth, environment, &env_lookup).await?;
     let outbound = litellm_inference::outbound::outbound_request(
         authenticated,
-        request.url.clone(),
-        &request.body,
+        url,
+        &body,
         Some(
-            request
-                .timeout
+            call.timeout
                 .unwrap_or(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)),
         ),
     )?;
@@ -47,8 +76,7 @@ pub async fn execute_audio_transcription_provider_call(
             error,
         ))
     })?;
-    Ok(request
-        .config
-        .transform_audio_transcription_response(&request.model, response_json)?
+    Ok(config
+        .transform_audio_transcription_response(provider.model, response_json)?
         .into_json())
 }

litellm-rust/crates/inference-transcription/src/lib.rs

@@ -2,16 +2,29 @@ pub mod types;
 pub use litellm_inference::RouteError as Error;
 mod constants;
 mod handler;
-mod prepare;
-pub use handler::execute_audio_transcription_provider_call;
+
 use litellm_auth::AuthServices;
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use litellm_inference::provider::resolve_llm_provider;
+use litellm_llms::{
+    base_llm::audio_transcription::transformation::BaseAudioTranscriptionConfig,
+    bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
+};
 use litellm_secrets::source::SecretSource;
-pub use prepare::prepare_audio_transcription_provider_call;
 use serde_json::Value;
 use std::sync::Arc;
 
 use crate::types::AudioTranscriptionRequest;
 
+fn audio_transcription_provider(
+    provider: LlmProviders,
+) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
+    match provider {
+        LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
+        _ => None,
+    }
+}
+
 #[derive(Clone)]
 pub struct AudioTranscriptionRoute {
     http: litellm_http::Client,
@@ -32,6 +45,10 @@ impl AudioTranscriptionRoute {
         }
     }
 
+    pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
+        self.run(request).await
+    }
+
     #[tracing::instrument(name = "litellm.route", skip_all, fields(
         route = "audio_transcription",
         model = request.model,
@@ -40,13 +57,23 @@ impl AudioTranscriptionRoute {
         stream = false,
         outcome
     ))]
-    pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
+    async fn run(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
         litellm_inference::diagnostic::unary(async {
-            let request =
-                prepare_audio_transcription_provider_call(request, self.secrets.as_ref()).await?;
-            litellm_inference::diagnostic::provider(&request.model, &request.custom_llm_provider);
+            let provider = resolve_llm_provider(
+                request.model,
+                request.custom_llm_provider,
+                "audio transcription",
+            )?;
+            let config = audio_transcription_provider(provider.provider).ok_or_else(|| {
+                Error::InvalidProvider(<&str>::from(provider.provider).to_string())
+            })?;
+            litellm_inference::diagnostic::provider(
+                provider.model,
+                <&'static str>::from(provider.provider),
+            );
+            let secrets = self.secrets.resolve(&config.secret_names()).await?;
             let execute: futures_util::future::BoxFuture<'_, Result<Value, Error>> = Box::pin(
-                execute_audio_transcription_provider_call(&self.http, &self.auth, request),
+                handler::execute(&self.http, &self.auth, config, provider, request, secrets),
             );
             execute.await
         })

litellm-rust/crates/inference-transcription/src/prepare.rs

@@ -1,65 +0,0 @@
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_http::request::string_headers;
-use litellm_http::request::with_default_headers;
-use litellm_llms::{
-    base_llm::{
-        audio_transcription::transformation::BaseAudioTranscriptionConfig,
-        auth::ValidatedEnvironment,
-    },
-    bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
-};
-use litellm_secrets::source::SecretSource;
-
-use super::Error;
-use crate::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest};
-use litellm_inference::provider::resolve_llm_provider;
-
-fn provider_config(provider: LlmProviders) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
-    match provider {
-        LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
-        _ => None,
-    }
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub async fn prepare_audio_transcription_provider_call(
-    request: AudioTranscriptionRequest<'_>,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderAudioTranscriptionRequest, Error> {
-    let provider_info = resolve_llm_provider(
-        request.model,
-        request.custom_llm_provider,
-        "audio transcription",
-    )?;
-    let model = provider_info.model.to_string();
-    let config = provider_config(provider_info.provider)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?;
-    let snapshot = secrets.resolve(&config.secret_names()).await?;
-    let env_lookup = |key: &str| snapshot.get(key);
-    let forwarded = string_headers("audio transcription", request.extra_headers)?;
-    let validated =
-        config.validate_environment(forwarded, &model, &request.optional_params, &env_lookup)?;
-    let environment = ValidatedEnvironment {
-        headers: with_default_headers(validated.headers, config.default_headers()),
-        auth: validated.auth,
-    };
-    let url = config.get_complete_url(
-        request.api_base,
-        &model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    let filtered_params = config.map_transcription_params(&request.optional_params);
-    let transformed =
-        config.transform_audio_transcription_request(&model, request.audio, filtered_params)?;
-    Ok(ProviderAudioTranscriptionRequest {
-        model,
-        custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
-        config,
-        url,
-        body: transformed.body,
-        environment,
-        secrets: snapshot,
-        timeout: request.timeout,
-    })
-}

litellm-rust/crates/inference-transcription/src/types.rs

@@ -1,9 +1,5 @@
-use litellm_secrets::source::Secrets;
 use std::time::Duration;
 
-use litellm_llms::base_llm::{
-    audio_transcription::transformation::BaseAudioTranscriptionConfig, auth::ValidatedEnvironment,
-};
 use serde_json::{Map, Value};
 
 pub struct AudioTranscriptionRequest<'a> {
@@ -16,37 +12,3 @@ pub struct AudioTranscriptionRequest<'a> {
     pub optional_params: Map<String, Value>,
     pub timeout: Option<Duration>,
 }
-
-#[derive(Clone)]
-pub struct ProviderAudioTranscriptionRequest {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseAudioTranscriptionConfig,
-    pub url: String,
-    pub body: Value,
-    pub environment: ValidatedEnvironment,
-    pub secrets: Secrets,
-    pub timeout: Option<Duration>,
-}
-
-impl ProviderAudioTranscriptionRequest {
-    pub fn model(&self) -> &str {
-        &self.model
-    }
-
-    pub fn custom_llm_provider(&self) -> &str {
-        &self.custom_llm_provider
-    }
-
-    pub fn url(&self) -> &str {
-        &self.url
-    }
-
-    pub fn body(&self) -> &Value {
-        &self.body
-    }
-
-    pub fn with_body(self, body: Value) -> Self {
-        Self { body, ..self }
-    }
-}

litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs

@@ -195,9 +195,10 @@ impl AnthropicBeta {
 
     fn rejected_by(&self) -> &'static [BetaProvider] {
         match self {
-            Self::AdvancedToolUse20251120 | Self::ContextManagement20250627 => {
-                &[BetaProvider::BedrockConverse]
+            Self::AdvancedToolUse20251120 => {
+                &[BetaProvider::BedrockConverse, BetaProvider::Databricks]
             }
+            Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
             Self::AdvisorTool20260301 | Self::Compact20260904 => &[
                 BetaProvider::AzureAi,
                 BetaProvider::BedrockConverse,
@@ -231,12 +232,11 @@ impl AnthropicBeta {
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::VertexAi,
+                BetaProvider::Databricks,
             ],
             Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
             Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
-            Self::DangerousToolUse20260903 => {
-                &[BetaProvider::BedrockConverse, BetaProvider::Databricks]
-            }
+            Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
             Self::Effort20251124 => &[BetaProvider::VertexAi],
             Self::FastMode20260201 | Self::Oauth20250420 => &[
                 BetaProvider::AzureAi,
@@ -244,6 +244,7 @@ impl AnthropicBeta {
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::VertexAi,
+                BetaProvider::Databricks,
             ],
             Self::FineGrainedToolStreaming20250514 => {
                 &[BetaProvider::AzureAi, BetaProvider::VertexAi]

litellm-rust/crates/llms/src/anthropic/beta_headers.rs

@@ -76,7 +76,7 @@ mod tests {
     #[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))]
     #[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))]
     #[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))]
-    #[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))]
+    #[case::databricks(BetaProvider::Databricks, None)]
     #[case::bedrock_converse(BetaProvider::BedrockConverse, None)]
     fn filter_renames_per_host(
         #[case] provider: BetaProvider,

litellm-rust/crates/llms/src/anthropic/messages/transformation.rs

@@ -74,6 +74,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -887,6 +888,7 @@ mod tests {
             headers(forwarded),
             api_key,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &env(vars),
         )
@@ -1206,6 +1208,7 @@ mod tests {
             Vec::new(),
             None,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &record,
         );

litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs

@@ -1,7 +1,7 @@
 use litellm_auth::SecretValue;
 use litellm_http::request::{has_bearer_auth, has_header};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
     providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
 };
 use litellm_router_types::LitellmParams;
@@ -83,6 +83,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -237,6 +238,7 @@ mod tests {
                     .collect(),
                 api_key,
                 "claude",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|_| None,
             )
@@ -622,6 +624,7 @@ mod tests {
             Vec::new(),
             None,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &record,
         );

litellm-rust/crates/llms/src/base_llm/messages/transformation.rs

@@ -1,4 +1,6 @@
-use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_llms_types::formats::messages::{
+    MessagesOptionalParams, MessagesRequest, MessagesResponse,
+};
 use litellm_router_types::LitellmParams;
 use serde_json::Value;
 
@@ -55,6 +57,7 @@ pub trait BaseMessagesConfig: Sync {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error>;
@@ -110,6 +113,7 @@ mod tests {
             headers: Headers,
             _api_key: Option<&str>,
             _model: &str,
+            _optional_params: &MessagesOptionalParams,
             _litellm_params: &LitellmParams,
             _env_lookup: &dyn Fn(&str) -> Option<String>,
         ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs

@@ -5,7 +5,7 @@ use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
 use litellm_core_utils::settings::resolve_non_empty;
 use litellm_http::request::{header_value, with_header, without_headers};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
     providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
 };
 use litellm_router_types::LitellmParams;
@@ -60,6 +60,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        optional_params: &MessagesOptionalParams,
         params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -67,11 +68,8 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
             .filter(|value| !value.trim().is_empty())
             .or_else(|| {
                 WORKSPACE_FIELDS.iter().find_map(|field| {
-                    params
-                        .extra
-                        .get(*field)?
-                        .as_str()
-                        .filter(|value| !value.trim().is_empty())
+                    workspace_value(optional_params.extra.get(*field))
+                        .or_else(|| workspace_value(params.extra.get(*field)))
                 })
             });
         let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS)
@@ -182,6 +180,10 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
     }
 }
 
+fn workspace_value(value: Option<&Value>) -> Option<&str> {
+    value?.as_str().filter(|value| !value.trim().is_empty())
+}
+
 fn platform_region(
     base: Option<&str>,
     params: &LitellmParams,
@@ -260,6 +262,7 @@ mod tests {
                     vec![],
                     Some("key"),
                     "claude-test",
+                    &MessagesOptionalParams::default(),
                     &LitellmParams::default(),
                     &|_| None
                 )
@@ -280,6 +283,7 @@ mod tests {
                 ],
                 Some("explicit"),
                 "claude-test",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|name| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into()),
             )
@@ -345,7 +349,14 @@ mod tests {
             .get_complete_url(None, "claude-test", &params, false, &env)
             .unwrap();
         let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
-            .validate_environment(vec![], None, "claude-test", &params, &env)
+            .validate_environment(
+                vec![],
+                None,
+                "claude-test",
+                &MessagesOptionalParams::default(),
+                &params,
+                &env,
+            )
             .unwrap();
         assert_eq!(
             url,

litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs

@@ -16,7 +16,7 @@ use litellm_auth_aws::{
     resolve_bedrock_region,
 };
 use litellm_llms_types::formats::messages::{
-    MessagesRequest,
+    MessagesOptionalParams, MessagesRequest,
     streaming::{MessagesStreamEvent, MessagesStreamUsage},
 };
 use litellm_llms_types::providers::bedrock::{
@@ -146,6 +146,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        _optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -618,6 +619,7 @@ mod tests {
                 Vec::new(),
                 None,
                 "anthropic.claude-3",
+                &MessagesOptionalParams::default(),
                 &litellm_params,
                 &|_| None,
             )
@@ -658,6 +660,7 @@ mod tests {
                 vec![("authorization".into(), "Bearer forwarded".into())],
                 api_key,
                 "anthropic.claude-3",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &env,
             )

litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs

@@ -6,7 +6,7 @@ use litellm_auth_aws::bedrock_model_id_and_region;
 use litellm_core_utils::settings::resolve_non_empty;
 use litellm_http::request::{with_header, without_headers};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
     providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
 };
 use litellm_router_types::LitellmParams;
@@ -59,6 +59,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        _optional_params: &MessagesOptionalParams,
         params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -317,7 +318,14 @@ mod tests {
             "https://private.test/anthropic/v1/messages"
         );
         let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
-            .validate_environment(vec![], None, model, &params, &|_| None)
+            .validate_environment(
+                vec![],
+                None,
+                model,
+                &MessagesOptionalParams::default(),
+                &params,
+                &|_| None,
+            )
             .unwrap();
         assert!(
             matches!(environment.auth, AuthScheme::AwsSigV4 { region: actual, service: "bedrock", .. } if actual == region)
@@ -347,7 +355,14 @@ mod tests {
             .get_complete_url(None, "claude-test", &params, false, &|_| None)
             .unwrap();
         let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
-            .validate_environment(vec![], None, "claude-test", &params, &|_| None)
+            .validate_environment(
+                vec![],
+                None,
+                "claude-test",
+                &MessagesOptionalParams::default(),
+                &params,
+                &|_| None,
+            )
             .unwrap();
         assert_eq!(
             url,
@@ -401,6 +416,7 @@ mod tests {
                 ],
                 key,
                 "claude-test",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|name| match name {
                     "BEDROCK_MANTLE_API_KEY" => Some("mantle".into()),

litellm-rust/crates/llms/src/deepseek/messages/transformation.rs

@@ -104,6 +104,7 @@ impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -298,6 +299,7 @@ mod tests {
                 .collect(),
             api_key,
             "deepseek-v4-pro",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             env,
         )

litellm-rust/crates/llms/src/edenai/messages/transformation.rs

@@ -36,6 +36,7 @@ impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/minimax/messages/transformation.rs

@@ -42,6 +42,7 @@ impl BaseMessagesConfig for MinimaxMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -183,7 +184,14 @@ mod tests {
                 .flatten()
         };
         let environment = MINIMAX_MESSAGES_CONFIG
-            .validate_environment(vec![], key, "test-model", &Default::default(), &env)
+            .validate_environment(
+                vec![],
+                key,
+                "test-model",
+                &MessagesOptionalParams::default(),
+                &Default::default(),
+                &env,
+            )
             .unwrap();
         assert!(matches!(
             environment.auth,

litellm-rust/crates/llms/src/openrouter/messages/transformation.rs

@@ -34,6 +34,7 @@ impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/tencent/messages/transformation.rs

@@ -43,6 +43,7 @@ impl BaseMessagesConfig for TencentAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -229,7 +230,14 @@ mod tests {
                 .flatten()
         };
         let environment = TENCENT_MESSAGES_CONFIG
-            .validate_environment(vec![], key, "test-model", &Default::default(), &env)
+            .validate_environment(
+                vec![],
+                key,
+                "test-model",
+                &MessagesOptionalParams::default(),
+                &Default::default(),
+                &env,
+            )
             .unwrap();
         assert!(matches!(
             environment.auth,

litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs

@@ -114,6 +114,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
         headers: Headers,
         _api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         _env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -439,6 +440,7 @@ mod tests {
                     .collect(),
                 Some("ignored-api-key"),
                 "claude-sonnet-4-5",
+                &MessagesOptionalParams::default(),
                 &params(Some("proj"), None),
                 &no_env,
             )

litellm-rust/crates/python-bridge/src/routes/responses/host.rs

@@ -1,22 +1,41 @@
 use std::convert::Infallible;
 
 use crate::routes::inference::InferenceHost;
-use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned};
-use litellm_inference_responses::{Error, route::Responses, types::ResponsesCall};
+use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
+use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned, from_py};
+use litellm_inference_responses::{
+    Error,
+    ids::{self, IdScope},
+    route::Responses,
+    types::ResponsesCall,
+};
 use pyo3::{
     gc::{PyTraverseError, PyVisit},
     prelude::*,
     types::PyDict,
 };
 
-pub(super) struct ResponsesPythonHost(pub InferenceHost);
+pub(super) struct ResponsesPythonHost {
+    host: InferenceHost,
+    ids: IdScope,
+}
+
+impl ResponsesPythonHost {
+    pub fn new(host: InferenceHost) -> Self {
+        Self {
+            host,
+            ids: IdScope::default(),
+        }
+    }
+}
 
 pub(super) fn project(
     host: &mut InferenceHost,
     py: Python<'_>,
     arguments: &Bound<'_, PyDict>,
 ) -> PyResult<ResponsesCall> {
-    let call = host.project(py, arguments, "input")?;
+    let mut call = host.project(py, arguments, "input")?;
+    ids::restore_input(&mut call.input);
     let optional_params = call
         .params
         .into_iter()
@@ -50,6 +69,39 @@ pub(super) fn project(
     })
 }
 
+/// The provider and deployment Python encodes into the ids it returns: the resolved provider and
+/// `litellm_metadata.model_info.id`
+fn id_scope(
+    host: &InferenceHost,
+    py: Python<'_>,
+    arguments: &Bound<'_, PyDict>,
+    call: &ResponsesCall,
+) -> PyResult<IdScope> {
+    let metadata = host.argument(py, arguments, "litellm_metadata")?;
+    let encrypted_content_affinity = match &metadata {
+        Some(metadata) => metadata
+            .call_method1("get", ("encrypted_content_affinity_enabled",))?
+            .is_truthy()?,
+        None => false,
+    };
+    let metadata = metadata
+        .map(|value| from_py::<serde_json::Value>(&value))
+        .transpose()?
+        .unwrap_or_default();
+    let model_id = match metadata.pointer("/model_info/id") {
+        None | Some(serde_json::Value::Null) => None,
+        Some(serde_json::Value::String(id)) => Some(id.clone()),
+        Some(other) => Some(other.to_string()),
+    };
+    Ok(IdScope {
+        provider: get_custom_llm_provider(&call.model, call.custom_llm_provider.as_deref())
+            .map(|resolved| resolved.custom_llm_provider.to_owned())
+            .or_else(|| call.custom_llm_provider.clone()),
+        model_id,
+        encrypted_content_affinity,
+    })
+}
+
 impl PythonBinding for ResponsesPythonHost {
     type Protocol = Responses;
     type Failure = PyErr;
@@ -59,7 +111,8 @@ impl PythonBinding for ResponsesPythonHost {
         py: Python<'_>,
         arguments: &Bound<'_, PyDict>,
     ) -> Result<ResponsesCall, InvokeError<Error>> {
-        let call = project(&mut self.0, py, arguments).map_err(InvokeError::Python)?;
+        let call = project(&mut self.host, py, arguments).map_err(InvokeError::Python)?;
+        self.ids = id_scope(&self.host, py, arguments, &call).map_err(InvokeError::Python)?;
         if call
             .optional_params
             .get("stream")
@@ -77,7 +130,9 @@ impl PythonBinding for ResponsesPythonHost {
         py: Python<'_>,
         response: <Responses as litellm_host::protocol::Protocol>::Response,
     ) -> PyResult<Py<PyAny>> {
-        self.0.response(py, &response)
+        let mut response = response;
+        ids::encode_response(&mut response, &self.ids);
+        self.host.response(py, &response)
     }
 
     fn encode_stream_head(
@@ -103,7 +158,7 @@ impl PythonBinding for ResponsesPythonHost {
     }
 
     fn map_error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
-        self.0.error(py, error)
+        self.host.error(py, error)
     }
     fn host_error(error: &PyErr) -> Error {
         Error::InvalidRequest(error.to_string().into())
@@ -122,9 +177,9 @@ impl PythonHostCalls<Responses> for ResponsesPythonHost {
 
 impl PythonOwned for ResponsesPythonHost {
     fn close(&mut self, _: Python<'_>) {
-        self.0.close();
+        self.host.close();
     }
     fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
-        self.0.traverse(visit)
+        self.host.traverse(visit)
     }
 }

litellm-rust/crates/python-bridge/src/routes/responses/mod.rs

@@ -62,7 +62,7 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py
         ));
     }
     let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE);
-    run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost(host))
+    run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost::new(host))
 }
 
 #[pyfunction]

litellm/__init__.py

@@ -7,6 +7,7 @@ _apply_process_defaults()
 ### INIT VARIABLES #########################
 import threading
 import os
+import datetime
 
 from collections.abc import Mapping, Sequence
 from typing import (
@@ -62,7 +63,6 @@ from litellm.constants import (
     modelscope_models,
     empower_models,
     together_ai_models,
-    baseten_models,
     WANDB_MODELS,
     REPEATED_STREAMING_CHUNK_LIMIT,
     request_timeout,
@@ -624,6 +624,7 @@ bedrock_models: Set = set()
 bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS)
 fal_ai_models: Set = set()
 fireworks_ai_models: Set = set()
+baseten_models: Set[str] = set()
 fireworks_ai_embedding_models: Set = set()
 deepinfra_models: Set = set()
 perplexity_models: Set = set()
@@ -676,6 +677,7 @@ qwen_ai_platform_models: Set = set()
 moonshot_models: Set = set()
 publicai_models: Set = set()
 darkbloom_models: Set = set()
+openinfer_models: Final[Set[str]] = set()
 v0_models: Set = set()
 morph_models: Set = set()
 lambda_ai_models: Set = set()
@@ -741,6 +743,16 @@ def is_openai_finetune_model(key: str) -> bool:
     return key.startswith("ft:") and not key.count(":") > 1
 
 
+def _is_past_deprecation_date(value: Mapping[str, object]) -> bool:
+    deprecation_date: Final = value.get("deprecation_date")
+    if not isinstance(deprecation_date, str):
+        return False
+    try:
+        return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today()
+    except ValueError:
+        return False
+
+
 def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
     for key, value in tuple(model_cost_map.items()):
         if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
@@ -841,6 +853,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
             if "-to-" not in key and "fireworks-ai-default" not in key:
                 fireworks_ai_models.add(key)
+        elif value.get("litellm_provider") == "baseten":
+            if _is_past_deprecation_date(value):
+                baseten_models.discard(key)
+            else:
+                baseten_models.add(key)
         elif value.get("litellm_provider") == "fireworks_ai-embedding-models":
             # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
             if "-to-" not in key:
@@ -945,6 +962,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             publicai_models.add(key)
         elif value.get("litellm_provider") == "darkbloom":
             darkbloom_models.add(key)
+        elif value.get("litellm_provider") == "openinfer":
+            openinfer_models.add(key)
         elif value.get("litellm_provider") == "v0":
             v0_models.add(key)
         elif value.get("litellm_provider") == "morph":
@@ -1113,6 +1132,7 @@ model_list = list(
     | moonshot_models
     | publicai_models
     | darkbloom_models
+    | openinfer_models
     | v0_models
     | morph_models
     | lambda_ai_models
@@ -1229,6 +1249,7 @@ def _build_models_by_provider() -> dict:
         "moonshot": moonshot_models,
         "publicai": publicai_models,
         "darkbloom": darkbloom_models,
+        "openinfer": openinfer_models,
         "v0": v0_models,
         "morph": morph_models,
         "lambda_ai": lambda_ai_models,

litellm/anthropic_beta_headers_config.json

@@ -220,36 +220,36 @@
   },
   "databricks": {
     "advisor-tool-2026-03-01": null,
-    "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
+    "advanced-tool-use-2025-11-20": null,
     "bash_20241022": null,
     "bash_20250124": null,
-    "code-execution-2025-08-25": "code-execution-2025-08-25",
+    "code-execution-2025-08-25": null,
     "compact-2026-01-12": "compact-2026-01-12",
     "computer-use-2025-01-24": "computer-use-2025-01-24",
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
-    "dangerous-tool-use-2026-09-03": null,
+    "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
-    "fast-mode-2026-02-01": "fast-mode-2026-02-01",
-    "files-api-2025-04-14": "files-api-2025-04-14",
+    "fast-mode-2026-02-01": null,
+    "files-api-2025-04-14": null,
     "structured-output-2024-03-01": null,
     "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
-    "mcp-client-2025-11-20": "mcp-client-2025-11-20",
-    "mcp-client-2025-04-04": "mcp-client-2025-04-04",
+    "mcp-client-2025-11-20": null,
+    "mcp-client-2025-04-04": null,
     "mcp-servers-2025-12-04": null,
-    "oauth-2025-04-20": "oauth-2025-04-20",
+    "oauth-2025-04-20": null,
     "output-128k-2025-02-19": "output-128k-2025-02-19",
     "per-turn-control-2026-07-01": null,
-    "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
-    "skills-2025-10-02": "skills-2025-10-02",
+    "prompt-caching-scope-2026-01-05": null,
+    "skills-2025-10-02": null,
     "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
     "text_editor_20241022": null,
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
-    "web-fetch-2025-09-10": "web-fetch-2025-09-10",
+    "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": "web-search-2025-03-05"
   }
 }

litellm/batches/batch_utils.py

@@ -794,11 +794,17 @@ def _batch_response_was_successful(
     message batch results lines report ``result.type == "succeeded"``; Bedrock
     batch output lines report ``modelOutput`` (and no ``error``).
     """
-    if custom_llm_provider == "anthropic":
-        return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
     if custom_llm_provider == "bedrock":
         return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None
-    _response: Final[dict[str, object]] = cast(  # cast-ok: batch response bodies come from provider JSON
-        dict[str, object], batch_job_output_file.get("response", None) or {}
+    envelope_key: Final = "result" if custom_llm_provider == "anthropic" else "response"
+    envelope: Final = batch_job_output_file.get(envelope_key)
+    if not isinstance(envelope, Mapping):
+        if envelope is not None:
+            verbose_logger.warning("batch output %s must be an object, treating the line as unsuccessful", envelope_key)
+        return False
+    response: Final = cast(Mapping[object, object], envelope)  # cast-ok: mapping shape validated above
+    return (
+        response.get("type") == "succeeded"
+        if custom_llm_provider == "anthropic"
+        else response.get("status_code") == 200
     )
-    return _response.get("status_code", None) == 200

litellm/constants.py

@@ -40,6 +40,14 @@ class ServerStreamingClassification(str, Enum):
 SERVER_STREAMING_CLASSIFICATION_MARKER: Final = ServerStreamingClassification.MARKER
 DEFER_PYDANTIC_BUILD: Final = os.getenv("DEFER_PYDANTIC_BUILD", "true") in ("true", "1", "on")
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
+KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS: Final = get_env_float(
+    "KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS", 5.0
+)
+KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS: Final = get_env_float(
+    "KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS", 300.0
+)
+KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS: Final = 2.0
+KUBERNETES_POD_ROUTING_KEY: Final = "kubernetes_pod_routing"
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
 AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"})
 ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5)
@@ -1030,6 +1038,7 @@ openai_compatible_endpoints: Final[list[str]] = [
     "https://api.prisminference.com/v1",
     "https://api.reka.ai/v1",
     "https://gigachat.devices.sberbank.ru/api/v1",
+    "https://api.openinfer.ai/v1",
 ]
 
 
@@ -1098,6 +1107,7 @@ openai_compatible_providers: Final[list] = [
     "docker_model_runner",
     "ragflow",
     "pinstripes",  # Pinstripes - JSON-configured provider
+    "openinfer",
     "darkbloom",
     "meta",  # Meta Model API (Muse Spark) - JSON-configured provider
     "cognition",
@@ -1254,14 +1264,6 @@ together_ai_models: Final[set] = set(
 # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
 
 
-baseten_models: Final[set] = set(
-    [
-        "qvv0xeq",
-        "q841o8w",
-        "31dxrj3",
-    ]
-)  # FALCON 7B  # WizardLM  # Mosaic ML
-
 featherless_ai_models: Final[set] = set(
     [
         "featherless-ai/Qwerky-72B",

litellm/cost_calculator.py

@@ -3113,14 +3113,19 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
         return response_ids[-1] if response_ids else None
 
     @staticmethod
-    def create_logging_object(results: Sequence[Mapping[str, object]]) -> LiteLLMRealtimeStreamLoggingObject:
+    def create_logging_object(
+        results: Sequence[Mapping[str, object]], litellm_model_name: str | None = None
+    ) -> LiteLLMRealtimeStreamLoggingObject:
         tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(results)
-        return LiteLLMRealtimeStreamLoggingObject(
+        logging_object: Final = LiteLLMRealtimeStreamLoggingObject(
             id=ResponsesWebSocketTokenUsageProcessor.response_id_from_responses_ws_results(results),
             usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(results),
             results=cast(OpenAIRealtimeStreamList, list(results)),  # cast-ok: raw WS frames, unvalidated field
             service_tier=next(iter(tier_partition)) if len(tier_partition) == 1 else None,
         )
+        if litellm_model_name is not None:
+            logging_object.hidden_params = {"litellm_model_name": litellm_model_name}
+        return logging_object
 
 
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"

litellm/decisions/call.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from functools import partial
@@ -6,6 +7,7 @@ from typing import Final, Literal, TypeAlias
 
 import httpx
 from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
+from pydantic_core import to_jsonable_python
 from typing_extensions import assert_never
 
 import litellm
@@ -56,6 +58,7 @@ from litellm.types.decisions import (
     OpenAIDecisionResponse,
     UnsupportedDecisionsRequest,
 )
+from litellm.types.llms.openai import ChatCompletionUserMessage
 from litellm.types.utils import LlmProviders, ProviderSpecificHeader
 from litellm.utils import ProviderConfigManager, get_model_cost_key
 
@@ -415,6 +418,30 @@ def _outbound_headers(
     return MappingProxyType({**(extra_headers or {}), **forwarded})
 
 
+_LOGGED_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+
+
+def _jsonable_fallback(value: object) -> object:
+    return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value)
+
+
+def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage:
+    caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python(
+        to_jsonable_python(caller_request, fallback=_jsonable_fallback)
+    )
+    sent_fields: Final = json.dumps(
+        dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {}))
+    )
+    return ChatCompletionUserMessage(role="user", content=sent_fields)
+
+
+def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None:
+    """Record every request field the caller sent as the logged `messages`, before any validation."""
+    logging_obj: Final = kwargs.get("litellm_logging_obj")
+    if isinstance(logging_obj, LiteLLMLoggingObj):
+        logging_obj.update_messages([_logged_request_message(caller_request)])
+
+
 def prepare_call(
     *,
     model: str,

litellm/decisions/main.py

@@ -17,6 +17,7 @@ from litellm.decisions.call import (
     SystemOneQuestions,
     SystemOneRequestFields,
     asend,
+    log_caller_request,
     prepare_call,
     reject_images_next_to_input,
     send,
@@ -78,6 +79,16 @@ def _prepare_decisions_call(
     extra_headers: Mapping[str, str] | None,
     kwargs: Mapping[str, object],
 ) -> DecisionsCall:
+    log_caller_request(
+        kwargs,
+        {
+            "input": decision_input,
+            "questions": questions,
+            "audio": audio,
+            "videos": videos,
+            **{key: kwargs[key] for key in _SYSTEMONE_FIELDS if key in kwargs},
+        },
+    )
     return prepare_call(
         model=model,
         fields=_request_fields(model, decision_input, questions, audio, videos, kwargs),

litellm/fine_tuning/main.py

@@ -173,6 +173,7 @@ def create_fine_tuning_job(
     try:
         _is_async: Final = kwargs.pop("acreate_fine_tuning_job", False) is True
         optional_params: Final = GenericLiteLLMParams(**kwargs)
+        provider_model: Final = model.removeprefix(f"{custom_llm_provider}/")
 
         # handle hyperparameters
         hyperparameters = hyperparameters or {}  # original hyperparameters
@@ -218,7 +219,7 @@ def create_fine_tuning_job(
             )
 
             create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
-                model,
+                provider_model,
                 training_file,
                 _oai_hyperparameters,
                 suffix,
@@ -263,7 +264,7 @@ def create_fine_tuning_job(
             extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams)
 
             create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
-                model,
+                provider_model,
                 training_file,
                 _oai_hyperparameters,
                 suffix,
@@ -298,7 +299,7 @@ def create_fine_tuning_job(
             response = vertex_fine_tuning_apis_instance.create_fine_tuning_job(
                 _is_async=_is_async,
                 create_fine_tuning_job_data=_build_fine_tuning_job_data(
-                    model,
+                    provider_model,
                     training_file,
                     _oai_hyperparameters,
                     suffix,

litellm/google_genai/main.py

@@ -16,6 +16,7 @@ from litellm.llms.base_llm.google_genai.transformation import (
     BaseGoogleGenAIGenerateContentConfig,
 )
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import CallTypes
@@ -194,12 +195,14 @@ class GenerateContentHelper:
         if litellm_logging_obj is None:
             raise ValueError("litellm_logging_obj is required, but got None")
 
+        vertex_location: Final = VertexBase.explicit_vertex_ai_location(dict(litellm_params))
         litellm_logging_obj.update_from_kwargs(
             kwargs=kwargs,
             model=model,
             optional_params=dict(generate_content_config_dict),
             litellm_params={
                 "litellm_call_id": litellm_call_id,
+                **({"vertex_location": vertex_location} if vertex_location else {}),
             },
             custom_llm_provider=custom_llm_provider,
         )

litellm/integrations/custom_guardrail.py

@@ -1139,6 +1139,17 @@ class CustomGuardrail(CustomLogger):
         """
         return False
 
+    def supports_structured_messages(self) -> bool:
+        """Whether this guardrail scans every row of ``structured_messages`` and returns texts that agree with them.
+
+        Guardrails that scan only part of the rows (the latest user turn, the
+        last row, the turns after the last assistant turn, or no tool results),
+        derive their returned texts from the rows, or cannot rewrite rows whose
+        content is a list of parts, override this to return False. Translation handlers that can rewrite the request from ``texts``
+        alone, like Bedrock Converse, then hand them only ``texts``.
+        """
+        return True
+
     def _suppressed_by_auto_router_compression(self) -> bool:
         """True when an auto router's own compression policy suppresses this guardrail.
 

litellm/integrations/custom_logger.py

@@ -912,9 +912,11 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         This is useful for logging payloads that contain sensitive information.
         """
         import litellm
-        from litellm import Choices, Message, ModelResponse
         from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS
-        from litellm.litellm_core_utils.redact_messages import redacted_litellm_params
+        from litellm.litellm_core_utils.redact_messages import (
+            redacted_litellm_params,
+            redacted_standard_logging_payload,
+        )
 
         turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False)
         excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
@@ -940,41 +942,14 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
             if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS)
         }
 
-        # Handle turn_off_message_logging - redact messages and responses (if not already excluded)
-        if turn_off_message_logging and not self.redacts_messages_itself():
-            redacted_str: Final = "redacted-by-litellm"
-
-            if "messages" not in (excluded_fields or ()) and standard_logging_object_copy.get("messages") is not None:
-                standard_logging_object_copy["messages"] = [Message(content=redacted_str).model_dump()]
-
-            if "response" not in (excluded_fields or ()) and standard_logging_object_copy.get("response") is not None:
-                response: Final = standard_logging_object_copy["response"]
-                # Check if this is a ResponsesAPIResponse (has "output" field)
-                if isinstance(response, dict) and "output" in response:
-                    # Make a copy to avoid modifying the original
-                    from copy import deepcopy
-
-                    response_copy: Final = deepcopy(response)
-                    # Redact content in output array
-                    if isinstance(response_copy.get("output"), list):
-                        for output_item in response_copy["output"]:
-                            if isinstance(output_item, dict) and "content" in output_item:
-                                if isinstance(output_item["content"], list):
-                                    # Redact text in content items
-                                    for content_item in output_item["content"]:
-                                        if isinstance(content_item, dict) and "text" in content_item:
-                                            content_item["text"] = redacted_str
-                    standard_logging_object_copy["response"] = response_copy
-                else:
-                    # Standard ModelResponse format
-                    model_response: Final = ModelResponse(choices=[Choices(message=Message(content=redacted_str))])
-                    model_response_dict: Final = model_response.model_dump()
-                    standard_logging_object_copy["response"] = model_response_dict
-
         return {
             **model_call_details,
             **redacted_params,
-            "standard_logging_object": standard_logging_object_copy,
+            "standard_logging_object": (
+                redacted_standard_logging_payload(standard_logging_object_copy)
+                if turn_off_message_logging and not self.redacts_messages_itself()
+                else standard_logging_object_copy
+            ),
         }
 
     async def get_proxy_server_request_from_cold_storage_with_object_key(

litellm/integrations/opentelemetry.py

@@ -13,6 +13,7 @@ from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.constants import KUBERNETES_POD_ROUTING_KEY
 from litellm.integrations._types.open_inference import (
     OpenInferenceSpanKindValues,
 )
@@ -50,6 +51,7 @@ from litellm.types.utils import (
     Function,
     LLMResponseTypes,
     StandardCallbackDynamicParams,
+    StandardLoggingMetadata,
     StandardLoggingPayload,
 )
 
@@ -2420,9 +2422,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             #############################################
             ############ LLM CALL METADATA ##############
             #############################################
-            metadata: Final = standard_logging_payload["metadata"]
+            metadata: Final = cast(  # cast-ok: payload metadata follows the standard logging contract
+                StandardLoggingMetadata, standard_logging_payload["metadata"]
+            )
             for key, value in metadata.items():
-                self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value)
+                self.safe_set_attribute(
+                    span=span,
+                    key=f"metadata.{key}",
+                    value=safe_dumps(value) if key == KUBERNETES_POD_ROUTING_KEY and value is not None else value,
+                )
             decision: Final = metadata.get("routing_decision")
             span.set_attributes(routing_decision_attributes(decision))
 

litellm/integrations/prometheus.py

@@ -47,9 +47,11 @@ from litellm.litellm_core_utils.core_helpers import (
     get_litellm_metadata_from_kwargs,
     get_metadata_variable_name_from_kwargs,
 )
+from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider
 from litellm.litellm_core_utils.service_tier_utils import (
     get_service_tier_from_standard_logging_payload,
 )
+from litellm.llms.openai_like.json_loader import JSONProviderRegistry
 from litellm.models.end_user import LiteLLM_EndUserTable
 from litellm.models.team import LiteLLM_TeamTable
 from litellm.models.user import LiteLLM_UserTable
@@ -60,6 +62,7 @@ from litellm.repositories.organization_repository import OrganizationRepository
 from litellm.repositories.table_repositories import EndUserRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
+from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.integrations.prometheus import *
 from litellm.types.integrations.prometheus import (
@@ -260,6 +263,26 @@ _POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, F
 _SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)])
 _DEFAULT_SERIES_CLEANUP_INTERVAL_SECONDS: Final = 60.0
 _ADDITIONAL_HEADERS: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
+_LOGGED_FIELDS: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_NO_LOGGED_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+
+
+def _logged_fields(value: object) -> Mapping[str, object]:
+    return _LOGGED_FIELDS.validate_python(value) if isinstance(value, Mapping) else _NO_LOGGED_FIELDS
+
+
+def _is_llm_provider(name: str) -> bool:
+    routable: Final = (
+        name in litellm.provider_list or JSONProviderRegistry.exists(name) or is_registered_custom_provider(name)
+    )
+    return routable and name not in litellm._known_custom_logger_compatible_callbacks  # pyright: ignore[reportPrivateUsage]  # the router treats these prefixes as prompt-management deployments
+
+
+def _deployment_model_provider(deployment: object) -> str:
+    if not isinstance(deployment, str):
+        return ""
+    prefix, separator, _ = deployment.partition("/")
+    return prefix if separator and _is_llm_provider(prefix) else ""
 
 
 def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
@@ -2846,6 +2869,25 @@ class PrometheusLogger(CustomLogger):
             )
             return None
 
+    @staticmethod
+    def _extract_deployment_api_provider(request_kwargs: Mapping[str, object]) -> str:
+        """
+        Provider of the deployment a failed call was routed to: the resolved
+        ``custom_llm_provider`` on ``litellm_params``, then the one recorded on
+        ``standard_logging_object``, then the provider prefix of the deployment
+        model the router wrote into the request metadata (``azure/my-gpt`` ->
+        ``azure``) when that prefix is an LLM provider the router accepts.
+        Returns an empty string otherwise.
+        """
+        litellm_params: Final = _logged_fields(request_kwargs.get("litellm_params"))
+        provider: Final = litellm_params.get("custom_llm_provider") or _logged_fields(
+            request_kwargs.get("standard_logging_object")
+        ).get("custom_llm_provider")
+        if isinstance(provider, str) and provider:
+            return provider
+        deployment_metadata: Final = find_deployment_metadata(litellm_params)
+        return _deployment_model_provider(deployment_metadata.get("deployment") if deployment_metadata else None)
+
     @staticmethod
     def _extract_api_provider_from_exception(exception: Exception) -> str | None:
         if not isinstance(exception, litellm.exceptions.RateLimitError):
@@ -3124,7 +3166,9 @@ class PrometheusLogger(CustomLogger):
                 label_litellm_model_name = litellm_model_name
                 label_model_id = model_id
                 label_api_base = api_base
-                label_api_provider = llm_provider
+                label_api_provider = self._extract_deployment_api_provider(
+                    _LOGGED_FIELDS.validate_python(request_kwargs)
+                )
                 label_requested_model = model_group or litellm_model_name
                 label_model_group = model_group
             else:

litellm/litellm_core_utils/completion_timeout.py

@@ -2,15 +2,25 @@
 
 from __future__ import annotations
 
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
+from enum import Enum
+from functools import partial
+from types import MappingProxyType
 from typing import Final
 
 import httpx
 from openai import Timeout as SDKTimeout
+from typing_extensions import assert_never
 
 from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS
 
 
+class StreamKind(Enum):
+    NOT_STREAMED = "not_streamed"
+    LIVE = "live"
+    FAKED = "faked"
+
+
 class CompletionTimeout:
     """Resolves HTTP timeout for ``completion()`` from model vs global settings."""
 
@@ -36,17 +46,91 @@ class CompletionTimeout:
             return COMPLETION_HTTP_FALLBACK_SECONDS
         return float(global_timeout)
 
+    @staticmethod
+    def _longer_limit(first: float | None, second: float | None) -> float | None:
+        return None if first is None or second is None else max(first, second)
+
+    @staticmethod
+    def _longer(first: float | httpx.Timeout, second: float | httpx.Timeout) -> float | httpx.Timeout:
+        if not isinstance(first, httpx.Timeout) and not isinstance(second, httpx.Timeout):
+            return max(first, second)
+        first_limits: Final = httpx.Timeout(first)
+        second_limits: Final = httpx.Timeout(second)
+        longer: Final = CompletionTimeout._longer_limit
+        return httpx.Timeout(
+            connect=longer(first_limits.connect, second_limits.connect),
+            read=longer(first_limits.read, second_limits.read),
+            write=longer(first_limits.write, second_limits.write),
+            pool=longer(first_limits.pool, second_limits.pool),
+        )
+
+    @staticmethod
+    def for_stream(
+        stream_kind: StreamKind, request_timeout: float | httpx.Timeout, stream_timeout: float | httpx.Timeout | None
+    ) -> float | httpx.Timeout:
+        """
+        ``stream_timeout`` bounds each read of a live stream, so it replaces the request timeout there. A fake
+        stream is one non-streaming request whose first byte arrives when generation ends, so it takes the longer
+        of each limit and never fails sooner than the same call would with either one alone.
+        """
+        if stream_timeout is None:
+            return request_timeout
+        match stream_kind:
+            case StreamKind.NOT_STREAMED:
+                return request_timeout
+            case StreamKind.LIVE:
+                return stream_timeout
+            case StreamKind.FAKED:
+                return CompletionTimeout._longer(request_timeout, stream_timeout)
+            case _:
+                assert_never(stream_kind)
+
+    @staticmethod
+    def as_limit(timeout: object) -> float | httpx.Timeout:
+        if isinstance(timeout, (httpx.Timeout, SDKTimeout)):
+            return CompletionTimeout.normalize(timeout)
+        if isinstance(timeout, (float, int, str)):
+            return float(timeout)
+        raise TypeError(f"timeout must be a number or an httpx.Timeout, got {type(timeout).__name__}")
+
+    @staticmethod
+    def stream_request_kwargs(non_stream_timeout: object, stream_timeout: object) -> Mapping[str, object]:
+        if stream_timeout is None:
+            return MappingProxyType({"timeout": non_stream_timeout})
+        return MappingProxyType(
+            {
+                "timeout": None
+                if non_stream_timeout is None
+                else CompletionTimeout.for_stream(
+                    StreamKind.FAKED,
+                    CompletionTimeout.as_limit(non_stream_timeout),
+                    CompletionTimeout.as_limit(stream_timeout),
+                ),
+                "stream_timeout": stream_timeout,
+            }
+        )
+
+    @staticmethod
+    def _coerce(
+        timeout: object, custom_llm_provider: str, supports_httpx_timeout: Callable[[str], bool]
+    ) -> float | httpx.Timeout:
+        limit: Final = CompletionTimeout.as_limit(timeout)
+        if not isinstance(limit, httpx.Timeout) or supports_httpx_timeout(custom_llm_provider):
+            return limit
+        return float(limit.read) if limit.read is not None else COMPLETION_HTTP_FALLBACK_SECONDS
+
     @staticmethod
     def resolve(
         model_timeout: float | str | httpx.Timeout | SDKTimeout | None,
-        kwargs: dict,
+        kwargs: Mapping[str, object],
         custom_llm_provider: str,
         *,
         global_timeout: float | str | None,
         supports_httpx_timeout: Callable[[str], bool],
+        stream_kind: StreamKind,
     ) -> float | httpx.Timeout:
         """
-        Resolution order (first non-None wins):
+        Request timeout resolution order (first non-None wins):
 
         1. ``model_timeout`` (call argument / merged ``litellm_params``)
         2. ``kwargs["timeout"]``
@@ -54,24 +138,20 @@ class CompletionTimeout:
         4. ``global_timeout`` (the explicitly-configured ``litellm.request_timeout``),
            or 600 when nothing was configured.
 
+        A stream then applies ``kwargs["stream_timeout"]`` per :meth:`for_stream`.
         Coerce :class:`httpx.Timeout` when the provider does not support it.
         """
-        resolved: float | str | httpx.Timeout | SDKTimeout
-        if model_timeout is not None:
-            resolved = model_timeout
-        elif kwargs.get("timeout") is not None:
-            resolved = kwargs["timeout"]
-        elif kwargs.get("request_timeout") is not None:
-            resolved = kwargs["request_timeout"]
-        else:
-            resolved = CompletionTimeout._fallback_when_no_explicit_timeout(global_timeout)
-
-        if isinstance(resolved, (httpx.Timeout, SDKTimeout)):
-            normalized: Final = CompletionTimeout.normalize(resolved)
-            if supports_httpx_timeout(custom_llm_provider):
-                return normalized
-            read_timeout: Final = normalized.read
-            return (
-                float(read_timeout) if read_timeout is not None else COMPLETION_HTTP_FALLBACK_SECONDS
-            )  # default 10 min timeout
-        return float(resolved)
+        coerce: Final = partial(
+            CompletionTimeout._coerce,
+            custom_llm_provider=custom_llm_provider,
+            supports_httpx_timeout=supports_httpx_timeout,
+        )
+        non_stream_candidates: Final = (model_timeout, kwargs.get("timeout"), kwargs.get("request_timeout"))
+        explicit: Final = next((candidate for candidate in non_stream_candidates if candidate is not None), None)
+        request_timeout: Final = coerce(
+            explicit if explicit is not None else CompletionTimeout._fallback_when_no_explicit_timeout(global_timeout)
+        )
+        stream_timeout: Final = None if stream_kind is StreamKind.NOT_STREAMED else kwargs.get("stream_timeout")
+        return CompletionTimeout.for_stream(
+            stream_kind, request_timeout, None if stream_timeout is None else coerce(stream_timeout)
+        )

litellm/litellm_core_utils/core_helpers.py

@@ -10,14 +10,16 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_c
 import httpx
 from pydantic import TypeAdapter, ValidationError
 
+import litellm
 from litellm._logging import verbose_logger
+from litellm.constants import KUBERNETES_POD_ROUTING_KEY
 from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
 from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    from litellm.types.utils import ModelResponseStream
+    from litellm.types.utils import ModelResponseStream, StandardLoggingKubernetesPodRouting
 
     Span = _Span | Any
 else:
@@ -92,6 +94,10 @@ def normalize_drop_params(value: object) -> bool | None:
         return None
 
 
+def drops_unsupported_params(drop_params: object) -> bool:
+    return litellm.drop_params is True or normalize_drop_params(drop_params) is True
+
+
 def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool:
     normalized: Final = normalize_drop_params(value)
     if normalized is None and value is not None:
@@ -385,6 +391,54 @@ def proxy_stamped_used_client_oauth_token(metadata: object, litellm_params: Mapp
     return metadata.get("used_client_oauth_token") if isinstance(metadata, Mapping) else None
 
 
+def proxy_stamped_kubernetes_pod_routing(
+    metadata: object,
+    litellm_params: Mapping[str, object] | None,
+) -> "StandardLoggingKubernetesPodRouting | None":
+    litellm_metadata: Final = litellm_params.get("litellm_metadata") if litellm_params is not None else None
+    litellm_routing: Final = _standard_logging_kubernetes_pod_routing(litellm_metadata)
+    if litellm_routing is not None:
+        return litellm_routing
+    return _standard_logging_kubernetes_pod_routing(metadata)
+
+
+def _standard_logging_kubernetes_pod_routing(
+    metadata: object,
+) -> "StandardLoggingKubernetesPodRouting | None":
+    if not isinstance(metadata, Mapping):
+        return None
+    metadata_mapping: Final = cast(  # cast-ok: routing metadata is a string-keyed mapping
+        Mapping[str, object], metadata
+    )
+    routing: Final = metadata_mapping.get(KUBERNETES_POD_ROUTING_KEY)
+    if not isinstance(routing, Mapping):
+        return None
+    routing_mapping: Final = cast(  # cast-ok: pod routing records use string keys
+        Mapping[str, object], routing
+    )
+    service_host: Final = routing_mapping.get("service_host")
+    pod_ip: Final = routing_mapping.get("pod_ip")
+    pod_count: Final = routing_mapping.get("pod_count")
+    selection: Final = routing_mapping.get("selection")
+    if (
+        not isinstance(service_host, str)
+        or not isinstance(pod_ip, str)
+        or not isinstance(pod_count, int)
+        or isinstance(pod_count, bool)
+        or pod_count < 1
+    ):
+        return None
+    if selection in ("round_robin", "session_affinity", "session_affinity_retry"):
+        routing_record: Final[StandardLoggingKubernetesPodRouting] = {
+            "service_host": service_host,
+            "pod_ip": pod_ip,
+            "pod_count": pod_count,
+            "selection": selection,
+        }
+        return routing_record
+    return None
+
+
 def get_litellm_metadata_from_kwargs(kwargs: dict):
     """
     Helper to get litellm metadata from all litellm request kwargs

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -2551,7 +2551,8 @@ def exception_type(
             ################################################################################
 
             if (
-                "Request Timeout Error" in error_str
+                isinstance(original_exception, httpx.TimeoutException)
+                or "Request Timeout Error" in error_str
                 or "Request timed out" in error_str
                 or "Timed out generating response" in error_str
                 or "The read operation timed out" in error_str
@@ -2559,7 +2560,7 @@ def exception_type(
                 exception_mapping_worked = True
 
                 raise Timeout(
-                    message=f"APITimeoutError - Request timed out. Error_str: {error_str}",
+                    message=f"APITimeoutError - Request timed out. Error_str: {error_str or type(mappable_exception).__name__}",
                     model=model,
                     llm_provider=custom_llm_provider,
                     litellm_debug_info=extra_information,

litellm/litellm_core_utils/get_litellm_params.py

@@ -142,14 +142,12 @@ def get_base_model_from_litellm_call_metadata(
     if metadata is None:
         return None
     model_info: Final = metadata.get("model_info")
-    if model_info:
-        model_info_mapping: Final = cast(  # cast-ok: model metadata is caller-provided and preserves its mapping shape
-            Mapping[str, object], model_info
-        )
-        return cast(  # cast-ok: model metadata values are caller-provided
-            str | None, model_info_mapping.get("base_model")
-        )
-    return None
+    if not model_info:
+        return None
+    base_model: Final = cast(  # cast-ok: model metadata is caller-provided and preserves its mapping shape
+        Mapping[str, object], model_info
+    ).get("base_model")
+    return base_model if isinstance(base_model, str) else None
 
 
 _get_base_model_from_litellm_call_metadata = get_base_model_from_litellm_call_metadata

litellm/litellm_core_utils/get_provider_specific_headers.py

@@ -9,6 +9,7 @@ class ProviderSpecificHeaderUtils:
     def get_provider_specific_headers(
         provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
         custom_llm_provider: str | None,
+        native_anthropic_messages: bool = False,
     ) -> Mapping[str, str]:
         """
         Get the provider specific headers for the given custom llm provider.
@@ -17,7 +18,8 @@ class ProviderSpecificHeaderUtils:
         carries its own comma-separated provider list, so headers that are safe for several
         providers and headers that are safe for exactly one can travel on the same request
         without sharing a scope. Entries whose provider list does not contain
-        `custom_llm_provider` contribute nothing.
+        `custom_llm_provider` contribute nothing, and entries marked
+        `native_anthropic_messages_only` contribute only to a provider's native Messages route.
 
         Returns:
             Dict: The provider specific headers for the given custom llm provider
@@ -31,6 +33,12 @@ class ProviderSpecificHeaderUtils:
 
         matched_headers: Final = {}
         for scoped_header in scoped_headers:
+            if (
+                "native_anthropic_messages_only" in scoped_header
+                and scoped_header["native_anthropic_messages_only"]
+                and not native_anthropic_messages
+            ):
+                continue
             stored_providers = scoped_header.get("custom_llm_provider", "")
             provider_list = [p.strip() for p in stored_providers.split(",")]
             if custom_llm_provider in provider_list:

litellm/litellm_core_utils/litellm_logging.py

@@ -19,7 +19,7 @@ from functools import lru_cache
 from types import MappingProxyType, TracebackType
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
 
-from httpx import Response
+from httpx import Response, Timeout
 from pydantic import BaseModel, JsonValue, TypeAdapter
 
 import litellm
@@ -43,6 +43,7 @@ from litellm.constants import (
     DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
     EMPTY_MAPPING,
     INTERNAL_CALLBACK_NAMES,
+    KUBERNETES_POD_ROUTING_KEY,
     PROVIDER_REQUEST_ID_HEADERS,
     REDACTED_BY_LITELLM,
 )
@@ -75,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import (
 from litellm.litellm_core_utils.core_helpers import (
     get_provider_response_headers_from_hidden_params,
     is_expected_client_error,
+    proxy_stamped_kubernetes_pod_routing,
     proxy_stamped_used_client_oauth_token,
     reconstruct_model_name,
     set_response_cost_in_hidden_params,
@@ -149,6 +151,7 @@ from litellm.types.llms.openai import (
 from litellm.types.mcp import MCPPostCallResponseObject
 from litellm.types.prompts.init_prompts import PromptSpec
 from litellm.types.rerank import RerankResponse
+from litellm.types.router import RouterRateLimitError
 from litellm.types.utils import (
     DEPLOYMENT_SCOPED_PRICING_FIELDS,
     CachingDetails,
@@ -298,7 +301,9 @@ else:
     _GENERIC_API_LOGGER_CLS: Final = GenericAPILogger
 _in_memory_loggers: Final[list[CustomLogger]] = []
 
-_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(("used_client_oauth_token", "usage_object"))
+_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(
+    ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY)
+)
 _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = (
     frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS
 )
@@ -634,15 +639,15 @@ class Logging(LiteLLMLoggingBaseClass):
         self,
         model: str,
         messages,
-        stream,
+        stream: bool | None,
         call_type,
         start_time: datetime.datetime,
         litellm_call_id: str,
         function_id: str,
         litellm_trace_id: str | None = None,
         dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = None,
-        dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = None,
-        dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = None,
+        dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
+        dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
         dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
         dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
         applied_guardrails: list[str] | None = None,
@@ -671,6 +676,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.message_copy_duration_ms: float = (time.time() - _copy_start) * 1000
         self.callback_duration_ms: float = 0.0
         self.stream = stream
+        self.applied_timeout: float | Timeout | None = None
         self.start_time = start_time  # log the call start time
         self.call_type = call_type
         self.relays_llm_passthrough: Final = call_type in (
@@ -717,8 +723,10 @@ class Logging(LiteLLMLoggingBaseClass):
 
         # Initialize dynamic callbacks
         self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks
-        self.dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = dynamic_success_callbacks
-        self.dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = (
+        self.dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
+            dynamic_success_callbacks
+        )
+        self.dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
             dynamic_async_success_callbacks
         )
         self.dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = dynamic_failure_callbacks
@@ -785,6 +793,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.defer_async_logging: bool = False
         self.enqueue_deferred_logging: Callable[[], None] | None = None
         self._async_success_scheduled: bool = False
+        self._sync_success_scheduled: bool = False
         self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
@@ -985,6 +994,16 @@ class Logging(LiteLLMLoggingBaseClass):
                 return model_id
         return None
 
+    def _routed_deployment_model(self) -> str | None:
+        litellm_params: Final = self._model_call_details_view.get("litellm_params")
+        metadata: Final = (
+            StandardLoggingPayloadSetup.merge_litellm_metadata(_LITELLM_PARAMS.validate_python(litellm_params))
+            if isinstance(litellm_params, dict)
+            else {}
+        )
+        deployment: Final = metadata.get("deployment")
+        return deployment if isinstance(deployment, str) else None
+
     def get_deployment_model_for_cost(self) -> str | None:
         """The provider-qualified model to price against.
 
@@ -1056,6 +1075,11 @@ class Logging(LiteLLMLoggingBaseClass):
         if "custom_llm_provider" in self.model_call_details:
             self.custom_llm_provider = self.model_call_details["custom_llm_provider"]
 
+    def record_applied_timeout(self, timeout: float | Timeout) -> None:
+        self.applied_timeout = timeout
+        self.litellm_params["timeout"] = timeout
+        self.model_call_details["litellm_params"] = self.litellm_params
+
     def update_from_kwargs(
         self,
         kwargs: dict,
@@ -2247,6 +2271,7 @@ class Logging(LiteLLMLoggingBaseClass):
             and litellm_params.get(CallTypes.agenerate_content.value, False) is not True
             and litellm_params.get(CallTypes.agenerate_content_stream.value, False) is not True
             and litellm_params.get(CallTypes.arealtime.value, False) is not True
+            and litellm_params.get(CallTypes.aresponses_websocket.value, False) is not True
         )
 
     _is_sync_litellm_request = is_sync_litellm_request
@@ -2312,7 +2337,7 @@ class Logging(LiteLLMLoggingBaseClass):
             **kwargs,
         )
 
-        if not self._should_run_sync_callbacks_for_async_calls():
+        if not self.stream or not self._should_run_sync_callbacks_for_async_calls():
             return
 
         executor.submit(
@@ -2440,7 +2465,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self,
         event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"],
     ) -> None:
-        if self.stream is not None and self.stream is True and event_type in ["async_success", "sync_success"]:
+        if self.stream is True and event_type in ["async_success", "sync_success"]:
             """
             Ignore check on stream, as there can be multiple chunks
             """
@@ -2502,7 +2527,8 @@ class Logging(LiteLLMLoggingBaseClass):
 
         elif self.call_type == CallTypes.aresponses_websocket.value and isinstance(result, list):  # pyright: ignore[reportUnknownMemberType]  # Logging.call_type is untyped
             logging_result = ResponsesWebSocketTokenUsageProcessor.create_logging_object(
-                results=result  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+                results=result,  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+                litellm_model_name=self._routed_deployment_model(),
             )
 
         elif self.relays_llm_passthrough and isinstance(result, Response):
@@ -2937,6 +2963,8 @@ class Logging(LiteLLMLoggingBaseClass):
         discarded as a duplicate of the in-progress create.
         """
         self.model_call_details.pop("has_logged_async_success", None)
+        self.model_call_details.pop("has_logged_sync_success", None)
+        self._sync_success_scheduled = False
         litellm_params = self.model_call_details.get("litellm_params")
         if not isinstance(litellm_params, dict):
             return
@@ -3125,6 +3153,11 @@ class Logging(LiteLLMLoggingBaseClass):
                 dynamic_success_callbacks=self.dynamic_success_callbacks,
                 global_callbacks=litellm.success_callback,
             )
+            delivery_callbacks: Final = (
+                self._get_completed_success_callbacks(asynchronous=False)
+                if for_async_request and not self.stream
+                else callbacks
+            )
 
             ## REDACT MESSAGES ##
             result = redact_message_input_output_from_logging(
@@ -3132,7 +3165,8 @@ class Logging(LiteLLMLoggingBaseClass):
                 result=result,
             )
             ## LOGGING HOOK ##
-            for callback in callbacks:
+            hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
+            for callback in hook_callbacks:
                 if isinstance(callback, CustomGuardrail):
                     from litellm.types.guardrails import GuardrailEventHooks
 
@@ -3162,7 +3196,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
             call_details_view: Final = MappingProxyType[str, object](call_details)
             self.mark_logging_complete(event_type="sync_success")
-            for callback in callbacks:
+            for callback in delivery_callbacks:
                 try:
                     should_run = self.should_run_callback(
                         callback=callback,
@@ -3434,7 +3468,12 @@ class Logging(LiteLLMLoggingBaseClass):
                             )
                     if (
                         isinstance(callback, CustomLogger)
-                        and is_sync_request
+                        and (is_sync_request or (for_async_request and not self.stream))
+                        and not (
+                            for_async_request
+                            and self.stream
+                            and callback in (self.dynamic_async_success_callbacks or ())
+                        )
                         and self.call_type
                         != CallTypes.pass_through.value  # pass-through endpoints call async_log_success_event
                     ):  # custom logger class
@@ -3451,7 +3490,7 @@ class Logging(LiteLLMLoggingBaseClass):
                                 result = call_details_view["complete_response"]
 
                             callback.log_success_event(
-                                kwargs=call_details,
+                                kwargs=callback.redact_standard_logging_payload_from_model_call_details(call_details),
                                 response_obj=result,
                                 start_time=start_time,
                                 end_time=end_time,
@@ -3693,6 +3732,9 @@ class Logging(LiteLLMLoggingBaseClass):
             dynamic_success_callbacks=self.dynamic_async_success_callbacks,
             global_callbacks=litellm._async_success_callback,
         )
+        delivery_callbacks: Final = (
+            self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks
+        )
 
         result = redact_message_input_output_from_logging(
             model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
@@ -3701,7 +3743,8 @@ class Logging(LiteLLMLoggingBaseClass):
 
         ## LOGGING HOOK ##
 
-        for callback in callbacks:
+        hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
+        for callback in hook_callbacks:
             try:
                 if isinstance(callback, CustomGuardrail):
                     from litellm.types.guardrails import GuardrailEventHooks
@@ -3739,7 +3782,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
         self.mark_logging_complete(event_type="async_success")
 
-        for callback in callbacks:
+        for callback in delivery_callbacks:
             # check if callback can run for this request
             litellm_params = self.model_call_details.get("litellm_params", {})
             should_run = self.should_run_callback(
@@ -3874,6 +3917,13 @@ class Logging(LiteLLMLoggingBaseClass):
                     traceback.format_exc(),
                 )
                 self._handle_callback_failure(callback=callback)
+        if not self.stream:
+            try:
+                self.handle_sync_success_callbacks_for_async_calls(
+                    result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit
+                )
+            except RuntimeError:
+                verbose_logger.exception("Could not schedule synchronous success callbacks")
         await flush_post_call_redis_batches()
 
     def _handle_callback_failure(self, callback: object):
@@ -4297,6 +4347,10 @@ class Logging(LiteLLMLoggingBaseClass):
         if self._should_run_sync_callbacks_for_async_calls() is False:
             return
 
+        if not self.stream and self._sync_success_scheduled:
+            return
+        if not self.stream:
+            self._sync_success_scheduled = True
         executor.submit(
             self.success_handler,
             result,
@@ -4315,11 +4369,57 @@ class Logging(LiteLLMLoggingBaseClass):
     ) -> None:
         executor.submit(self.success_handler, result, start_time, end_time, cache_hit)
 
+    def _get_completed_success_callbacks(
+        self, *, asynchronous: bool
+    ) -> tuple[str | Callable[..., object] | CustomLogger, ...]:
+        sync_callbacks: Final = self.get_combined_callback_list(
+            self.dynamic_success_callbacks, litellm.success_callback
+        )
+        async_callbacks: Final = self.get_combined_callback_list(
+            self.dynamic_async_success_callbacks,
+            litellm._async_success_callback,  # pyright: ignore[reportPrivateUsage]  # shared logging registry
+        )
+        legacy_sync_callbacks: Final = self._remove_internal_litellm_callbacks(
+            self._remove_internal_custom_logger_callbacks(sync_callbacks)
+        )
+        selected: Final = async_callbacks if asynchronous else sync_callbacks
+        other: Final = sync_callbacks if asynchronous else async_callbacks
+        return tuple(
+            callback
+            for callback in self.get_combined_callback_list(selected, other)
+            if (
+                (
+                    (
+                        getattr(callback.async_log_success_event, "__func__", callback.async_log_success_event)
+                        is not CustomLogger.async_log_success_event
+                    )
+                    == asynchronous
+                    and (
+                        asynchronous
+                        or getattr(callback.log_success_event, "__func__", callback.log_success_event)
+                        is not CustomLogger.log_success_event
+                    )
+                )
+                if isinstance(callback, CustomLogger)
+                else callback in selected
+                and (
+                    asynchronous
+                    or (
+                        callback not in async_callbacks
+                        and callback in legacy_sync_callbacks
+                        and not (isinstance(callback, str) and callback in INTERNAL_CALLBACK_NAMES)
+                    )
+                )
+            )
+        )
+
     def _should_run_sync_callbacks_for_async_calls(self) -> bool:
         """
         Returns:
             - bool: True if sync callbacks should be run for async calls. eg. `langfuse`, `s3`
         """
+        if not self.stream:
+            return bool(self._get_completed_success_callbacks(asynchronous=False))
         _combined_sync_callbacks: Final = self.get_combined_callback_list(
             dynamic_success_callbacks=self.dynamic_success_callbacks,
             global_callbacks=litellm.success_callback,
@@ -4349,7 +4449,11 @@ class Logging(LiteLLMLoggingBaseClass):
         _filtered_failure_callbacks = self._remove_internal_litellm_callbacks(_filtered_failure_callbacks)
         return len(_filtered_failure_callbacks) > 0
 
-    def get_combined_callback_list(self, dynamic_success_callbacks: list | None, global_callbacks: list) -> list:
+    def get_combined_callback_list(
+        self,
+        dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None,
+        global_callbacks: list[str | Callable[..., object] | CustomLogger],
+    ) -> list[str | Callable[..., object] | CustomLogger]:
         if dynamic_success_callbacks is None:
             return list(global_callbacks)
         return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
@@ -6176,7 +6280,7 @@ class StandardLoggingPayloadSetup:
     @staticmethod
     def get_standard_logging_metadata(
         metadata: Mapping[str, object] | None,
-        litellm_params: dict | None = None,
+        litellm_params: dict[str, object] | None = None,
         prompt_integration: str | None = None,
         applied_guardrails: list[str] | None = None,
         mcp_tool_call_metadata: StandardLoggingMCPToolCall | None = None,
@@ -6216,6 +6320,8 @@ class StandardLoggingPayloadSetup:
                     prompt_integration=prompt_integration,
                 )
 
+        kubernetes_pod_routing: Final = proxy_stamped_kubernetes_pod_routing(metadata, litellm_params)
+
         # Initialize with default values
         clean_metadata = StandardLoggingMetadata(
             user_api_key_hash=None,
@@ -6252,6 +6358,7 @@ class StandardLoggingPayloadSetup:
             user_api_key_auth_metadata=None,
             team_alias=None,
             team_id=None,
+            kubernetes_pod_routing=kubernetes_pod_routing,
             used_client_oauth_token=resolve_used_client_oauth_token(
                 proxy_stamped_used_client_oauth_token(metadata, litellm_params),
                 custom_llm_provider,
@@ -6364,6 +6471,7 @@ class StandardLoggingPayloadSetup:
         custom_llm_provider: str | None,
         init_response_obj: object,
         api_base: str | None = None,
+        router_model_id: str | None = None,
     ) -> StandardLoggingModelInformation:
         model_cost_name: Final = select_model_name_for_cost_calc(
             model=base_model if custom_pricing else None,
@@ -6371,12 +6479,23 @@ class StandardLoggingPayloadSetup:
             base_model=base_model,
             custom_pricing=custom_pricing,
         )
+        pricing_model_name: Final = (
+            select_model_name_for_cost_calc(
+                model=base_model if custom_pricing else None,
+                completion_response=init_response_obj,
+                base_model=base_model,
+                custom_pricing=custom_pricing,
+                router_model_id=router_model_id,
+            )
+            if router_model_id is not None
+            else model_cost_name
+        )
         if model_cost_name is None:
             model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None)
         else:
             try:
                 _model_cost_information: Final = litellm.get_model_info(
-                    model=model_cost_name,
+                    model=pricing_model_name or model_cost_name,
                     custom_llm_provider=custom_llm_provider,
                     api_base=api_base,
                 )
@@ -6586,6 +6705,7 @@ class StandardLoggingPayloadSetup:
         rate_limit_category: Final = validate_rate_limit_category(getattr(original_exception, "category", None))
         rate_limit_type: Final = validate_rate_limit_type(getattr(original_exception, "rate_limit_type", None))
         budget_error: Final = original_exception if isinstance(original_exception, BudgetExceededError) else None
+        router_error: Final = original_exception if isinstance(original_exception, RouterRateLimitError) else None
         provider_request_id: Final = _get_provider_request_id(original_exception) if original_exception else None
 
         return StandardLoggingPayloadErrorInformation(
@@ -6601,6 +6721,7 @@ class StandardLoggingPayloadSetup:
             error_budget_entity_id=budget_error.entity_id if budget_error else None,
             error_budget_limit=budget_error.max_budget if budget_error else None,
             error_budget_spend=budget_error.current_cost if budget_error else None,
+            error_cooldown_deployments=router_error.cooldown_deployments if router_error else None,
             normalized_error=normalize_error(original_exception, error_status, error_message),
         )
 
@@ -7084,6 +7205,7 @@ def get_standard_logging_object_payload(
             custom_llm_provider=kwargs.get("custom_llm_provider"),
             init_response_obj=init_response_obj,
             api_base=litellm_params.get("api_base"),
+            router_model_id=logging_obj.get_router_model_id(),
         )
 
         error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
@@ -7319,6 +7441,7 @@ def get_standard_logging_metadata(
         user_api_key_auth_metadata=None,
         team_alias=None,
         team_id=None,
+        kubernetes_pod_routing=None,
         used_client_oauth_token=None,
     )
     if isinstance(metadata, dict):
@@ -7392,6 +7515,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
         requester_ip_address="127.0.0.1",
         requester_metadata=None,
         user_api_key_end_user_id="test_end_user",
+        kubernetes_pod_routing=None,
     )
 
     hidden_params: Final = StandardLoggingHiddenParams(

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1088,6 +1088,25 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
     )
 
 
+def _modality_token_cost(
+    model_info: ModelInfo,
+    modality_cost_key: str,
+    tokens: int,
+    prompt_base_cost: float,
+    service_tier: str | None,
+) -> float:
+    """
+    Audio, image and video prompt tokens bill at their modality rate when the model has one.
+    Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which
+    `_get_token_base_cost` has already resolved for the service tier and long-context tier.
+    """
+    tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier)
+    cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key
+    if model_info.get(cost_key) is None:
+        return float(tokens) * prompt_base_cost
+    return calculate_cost_component(model_info, cost_key, tokens)
+
+
 def _calculate_input_cost(
     prompt_tokens_details: PromptTokensDetailsResult,
     model_info: ModelInfo,
@@ -1118,28 +1137,37 @@ def _calculate_input_cost(
     if prompt_tokens_details["audio_tokens"] and not (
         prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
     ):
-        audio_cost_key: Final = get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
-        prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
+        prompt_cost += _modality_token_cost(
+            model_info,
+            "input_cost_per_audio_token",
+            prompt_tokens_details["audio_tokens"],
+            prompt_base_cost,
+            service_tier,
+        )
 
     ### IMAGE TOKEN COST
     if prompt_tokens_details["image_tokens"] and not (
         prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
     ):
-        # For image token costs:
-        # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
-        image_token_cost_key = "input_cost_per_image_token"
-        if model_info.get(image_token_cost_key) is None:
-            image_token_cost_key = "input_cost_per_token"
-        prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
+        prompt_cost += _modality_token_cost(
+            model_info,
+            "input_cost_per_image_token",
+            prompt_tokens_details["image_tokens"],
+            prompt_base_cost,
+            service_tier,
+        )
 
     ### VIDEO TOKEN COST
     if prompt_tokens_details["video_tokens"] and not (
         prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
     ):
-        video_token_cost_key = "input_cost_per_video_token"
-        if model_info.get(video_token_cost_key) is None:
-            video_token_cost_key = "input_cost_per_token"
-        prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"])
+        prompt_cost += _modality_token_cost(
+            model_info,
+            "input_cost_per_video_token",
+            prompt_tokens_details["video_tokens"],
+            prompt_base_cost,
+            service_tier,
+        )
 
     ### CACHE WRITING COST - Now uses tiered pricing
     if (

litellm/litellm_core_utils/prompt_templates/chat_template_tools.py

@@ -0,0 +1,37 @@
+import json
+import re
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Final
+
+
+@dataclass(frozen=True, slots=True)
+class ChatTemplateToolSection:
+    preamble_tokens: int
+    implicit_system_turn_tokens: int
+    render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
+
+
+def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
+    return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions)
+
+
+def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
+    return "".join(
+        "\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions
+    )
+
+
+_TOOL_SECTIONS: Final = (
+    (
+        re.compile(r"deepseek-v4(?![.\d]|p\d)", re.IGNORECASE),
+        ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas),
+    ),
+    (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
+)
+
+
+def chat_template_tool_section(model: str | None) -> ChatTemplateToolSection | None:
+    if not model:
+        return None
+    return next((section for pattern, section in _TOOL_SECTIONS if pattern.search(model)), None)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1669,7 +1669,7 @@ def _replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Patte
     return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK
 
 
-def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
+def sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
     """Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$."""
     return _replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS)
 
@@ -1815,7 +1815,7 @@ def convert_to_anthropic_tool_result(
         tool_message: Final[ChatCompletionToolMessage] = message
         tool_call_id: str = tool_message["tool_call_id"]
         # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$
-        sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id)
+        sanitized_tool_use_id = sanitize_anthropic_tool_use_id(tool_call_id)
 
         # We can't determine from openai message format whether it's a successful or
         # error call result so default to the successful result template
@@ -1829,7 +1829,7 @@ def convert_to_anthropic_tool_result(
         function_message: Final[ChatCompletionFunctionMessage] = message
         tool_call_id = function_message.get("tool_call_id") or str(uuid.uuid4())
         # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$
-        sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id)
+        sanitized_tool_use_id = sanitize_anthropic_tool_use_id(tool_call_id)
         anthropic_tool_result = AnthropicMessagesToolResultParam(
             type="tool_result",
             tool_use_id=sanitized_tool_use_id,
@@ -2037,7 +2037,7 @@ def convert_to_anthropic_tool_invoke(
         elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
             continue
         else:
-            sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id)
+            sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
             _anthropic_tool_use_param = AnthropicMessagesToolUseParam(
                 type="tool_use",
                 id=sanitized_tool_id,
@@ -5533,6 +5533,11 @@ def _bedrock_tools_pt(
     return tool_block_list
 
 
+def bedrock_tool_blocks(tools: Sequence[Mapping[str, object]], model: str | None) -> tuple[BedrockToolBlock, ...]:
+    """Bedrock Converse ``toolConfig.tools`` entries for OpenAI or Anthropic tool definitions."""
+    return tuple(_bedrock_tools_pt([dict(tool) for tool in tools], model=model))
+
+
 def _is_system_message(message: Mapping[str, object]) -> bool:
     role: Final = message.get("role")
     return isinstance(role, str) and "system" in role
@@ -5895,21 +5900,21 @@ def _optional_tool_name(value: object) -> str | None:
 
 
 def _split_tool_call_ids(calls: Sequence[tuple[str | None, int]]) -> tuple[tuple[str | None, ...], ...]:
-    taken: Final = frozenset(_sanitize_anthropic_tool_use_id(call_id) for call_id, _ in calls if call_id)
+    taken: Final = frozenset(sanitize_anthropic_tool_use_id(call_id) for call_id, _ in calls if call_id)
 
     def fresh(call_id: str) -> Iterator[str]:
         return filter(
-            lambda candidate: _sanitize_anthropic_tool_use_id(candidate) not in taken,
+            lambda candidate: sanitize_anthropic_tool_use_id(candidate) not in taken,
             (f"{call_id}__concat_{n}" for n in itertools.count(1)),
         )
 
     suffixes: Final = MappingProxyType(
-        {_sanitize_anthropic_tool_use_id(call_id): fresh(call_id) for call_id, count in calls if call_id and count > 1}
+        {sanitize_anthropic_tool_use_id(call_id): fresh(call_id) for call_id, count in calls if call_id and count > 1}
     )
     return tuple(
         (
             call_id,
-            *(next(suffixes[_sanitize_anthropic_tool_use_id(call_id)]) for _ in range(count - 1)),
+            *(next(suffixes[sanitize_anthropic_tool_use_id(call_id)]) for _ in range(count - 1)),
         )
         if call_id
         else (None,) * count
@@ -6166,3 +6171,25 @@ def resolve_structured_messages(
         if structured:
             return [msg if isinstance(msg, dict) else msg.model_dump() for msg in structured]
     return None
+
+
+def is_flat_responses_tool(tool: Mapping[str, object]) -> bool:
+    tool_type: Final = tool.get("type")
+    return tool_type in ("function", "custom") and tool_type not in tool and "input_schema" not in tool
+
+
+def chat_completion_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
+    from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
+    from litellm.responses.litellm_completion_transformation.transformation import (
+        LiteLLMCompletionResponsesConfig,
+    )
+
+    if is_flat_responses_tool(tool):
+        chat_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
+            [dict(tool)]  # pyright: ignore[reportArgumentType]  # a JSON dict in a flat Responses tool shape
+        )
+        return tuple(chat_tools)
+    anthropic_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(
+        [dict(tool)]  # pyright: ignore[reportArgumentType]  # a JSON dict in an Anthropic or chat tool shape
+    )
+    return tuple(anthropic_tools)

litellm/litellm_core_utils/request_timeout_resolver.py

@@ -12,10 +12,15 @@ tell "user asked for this" from "nobody set it". This resolver answers that:
 
 from __future__ import annotations
 
-from typing import Final
+from collections.abc import Mapping
+from typing import Final, Literal, Protocol
+
+from pydantic import TypeAdapter
 
 from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
 
+_TIMEOUTS_BY_SERVICE_TIER: Final[TypeAdapter[Mapping[str, float] | None]] = TypeAdapter(Mapping[str, float] | None)
+
 
 def get_configured_request_timeout() -> float | None:
     """Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
@@ -27,3 +32,19 @@ def get_configured_request_timeout() -> float | None:
     if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
         return timeout
     return None
+
+
+class _DeploymentParams(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
+def get_service_tier_timeout(
+    service_tier: object,
+    litellm_params: _DeploymentParams,
+    key: Literal["timeout_by_service_tier", "stream_timeout_by_service_tier"],
+) -> float | None:
+    """Return the deployment's ``key`` timeout for the request's ``service_tier``, else ``None``."""
+    if not isinstance(service_tier, str):
+        return None
+    timeouts: Final = _TIMEOUTS_BY_SERVICE_TIER.validate_python(litellm_params.get(key))
+    return None if timeouts is None else timeouts.get(service_tier)

litellm/litellm_core_utils/streaming_handler.py

@@ -45,7 +45,7 @@ from litellm.types.utils import (
 )
 from litellm.types.utils import GenericStreamingChunk as GChunk
 
-from ..exceptions import OpenAIError
+from ..exceptions import OpenAIError, Timeout
 from .core_helpers import map_finish_reason, process_response_headers
 from .exception_mapping_utils import exception_type, with_deployment_api_base
 from .llm_response_utils.get_api_base import get_api_base
@@ -88,6 +88,16 @@ def _apply_provider_fields(response: ModelResponseStream, fields: Mapping[str, o
         setattr(response, key, value)
 
 
+def _is_transport_timeout(error: BaseException) -> bool:
+    return isinstance(error, httpx.TimeoutException) or (
+        isinstance(error, Timeout) and isinstance(error.__cause__, httpx.TimeoutException)
+    )
+
+
+def _skips_stream_fallback(status_code: int | None) -> bool:
+    return status_code is not None and 400 <= status_code < 500 and status_code != 429
+
+
 def is_async_iterable(obj: object) -> bool:
     """
     Check if an object is an async iterable (can be used with 'async for').
@@ -2384,6 +2394,8 @@ class CustomStreamWrapper:
 
         429 (rate-limit) is explicitly exempted from the 4xx filter because
         it is transient and the Router should switch to another model group.
+        So is a transport read timeout, which maps to a 408 Timeout, including
+        one an inner Responses stream already mapped for the chat bridge.
 
         An error an inner stream already wrapped (the chat-to-Responses bridge
         consumes a Responses stream) is rebuilt around the provider exception
@@ -2448,9 +2460,9 @@ class CustomStreamWrapper:
         # Raise non-retriable client errors directly (skip fallback).
         # Exception: 429 (rate-limit) IS retriable/transient — allow it
         # through so the Router can switch to a different model group.
-        if mapped_status_code is not None and 400 <= mapped_status_code < 500 and mapped_status_code != 429:
-            raise mapped_exception
-        if original_status_code is not None and 400 <= original_status_code < 500 and original_status_code != 429:
+        if not _is_transport_timeout(e) and (
+            _skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code)
+        ):
             raise mapped_exception
 
         raise MidStreamFallbackError(

litellm/litellm_core_utils/token_counter.py

@@ -35,6 +35,10 @@ from litellm.constants import (
     TOKEN_COUNTER_MAX_EXACT_CHARS,
 )
 from litellm.litellm_core_utils.asyncify import asyncify
+from litellm.litellm_core_utils.prompt_templates.chat_template_tools import (
+    ChatTemplateToolSection,
+    chat_template_tool_section,
+)
 from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding
 
 if TYPE_CHECKING:
@@ -347,6 +351,9 @@ EXTRAPOLATION_SAMPLES: Final = 16
 T_ParamSpec: Final = ParamSpec("T_ParamSpec")
 T_Retval = TypeVar("T_Retval")
 _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter")
+_ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset(
+    ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming")
+)
 
 
 def _count_offload_limiter_for_this_loop() -> anyio.CapacityLimiter:
@@ -404,6 +411,7 @@ class _MessageCountParams:
             self.tokens_per_message = 3
             self.tokens_per_name = 1
         self.count_function = _get_count_function(model, custom_tokenizer)
+        self.tool_section = chat_template_tool_section(model)
 
 
 def token_counter(
@@ -467,7 +475,9 @@ def token_counter(
         num_tokens = _count_messages(params, new_messages, use_default_image_token_count, default_token_count)
         if count_response_tokens is False:
             includes_system_message: Final = any(message.get("role", None) == "system" for message in new_messages)
-            num_tokens += _count_extra(params.count_function, tools, tool_choice, includes_system_message)
+            num_tokens += _count_extra(
+                params.count_function, tools, tool_choice, includes_system_message, params.tool_section
+            )
 
     else:
         raise ValueError("Either text or messages must be provided")
@@ -501,7 +511,8 @@ def messages_reach_token_count(
         _count_messages(params, [message], use_default_image_token_count, None) for message in new_messages
     )
     running_totals: Final = accumulate(
-        per_message_counts, initial=_count_extra(params.count_function, tools, None, includes_system_message)
+        per_message_counts,
+        initial=_count_extra(params.count_function, tools, None, includes_system_message, params.tool_section),
     )
     return any(total >= threshold for total in running_totals)
 
@@ -592,6 +603,7 @@ def _count_extra(
     tools: list[ChatCompletionToolParam] | None,
     tool_choice: ChatCompletionNamedToolChoiceParam | None,
     includes_system_message: bool,
+    tool_section: ChatTemplateToolSection | None,
 ) -> int:
     """Count extra tokens for function definitions and tool choices.
     Args:
@@ -599,26 +611,56 @@ def _count_extra(
         tools (Optional[List[ChatCompletionToolParam]]): The available tools.
         tool_choice (Optional[ChatCompletionNamedToolChoiceParam]): The tool choice.
         includes_system_message (bool): Whether the messages include a system message.
+        tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known.
     """
+    reply_primer_tokens: Final = 3
+    tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice)
+    return (
+        reply_primer_tokens
+        + _count_tool_definitions(count_function, tools, includes_system_message, tool_section)
+        + tool_choice_tokens
+    )
 
-    num_tokens = 3  # every reply is primed with <|start|>assistant<|message|>
 
-    if tools:
-        num_tokens += count_function(_format_function_definitions(tools))
-        num_tokens += 9  # Additional tokens for function definition of tools
-    # If there's a system message and tools are present, subtract four tokens
-    if tools and includes_system_message:
-        num_tokens -= 4
-    # If tool_choice is 'none', add one token.
-    # If it's an object, add 4 + the number of tokens in the function name.
-    # If it's undefined or 'auto', don't add anything.
+def _count_tool_definitions(
+    count_function: TokenCounterFunction,
+    tools: Sequence[object] | None,
+    includes_system_message: bool,
+    tool_section: ChatTemplateToolSection | None,
+) -> int:
+    if not tools:
+        return 0
+    if tool_section is not None:
+        return _count_chat_template_tools(count_function, tools, includes_system_message, tool_section)
+    system_message_discount: Final = 4 if includes_system_message else 0
+    return count_function(_format_function_definitions(tools)) + 9 - system_message_discount
+
+
+def _count_tool_choice(
+    count_function: TokenCounterFunction,
+    tool_choice: ChatCompletionNamedToolChoiceParam | None,
+) -> int:
     if tool_choice == "none":
-        num_tokens += 1
-    elif isinstance(tool_choice, dict):
-        num_tokens += 7
-        num_tokens += count_function(str(tool_choice["function"]["name"]))
+        return 1
+    if isinstance(tool_choice, dict):
+        return 7 + count_function(str(tool_choice["function"]["name"]))
+    return 0
 
-    return num_tokens
+
+def _count_chat_template_tools(
+    count_function: TokenCounterFunction,
+    tools: Sequence[object],
+    includes_system_message: bool,
+    tool_section: ChatTemplateToolSection,
+) -> int:
+    functions: Final = tuple(
+        function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name")
+    )
+    if not functions:
+        return 0
+    system_turn_tokens: Final = 0 if includes_system_message else tool_section.implicit_system_turn_tokens
+    schemas: Final = tool_section.render_tool_schemas(functions)
+    return tool_section.preamble_tokens + system_turn_tokens + count_function(schemas)
 
 
 def _get_extrapolating_count_function(
@@ -1034,15 +1076,31 @@ def _format_function_definitions(tools: Sequence[object]) -> str:
     lines: Final = []
     lines.append("namespace functions {")
     lines.append("")
-    for tool in tools:
-        if not isinstance(tool, Mapping):
-            continue
-        for function in _function_definitions_for_tool(cast(Mapping[str, object], tool)):
-            lines.extend(_format_single_function_definition(function))
+    for function in _function_definitions(tools):
+        lines.extend(_format_single_function_definition(function))
     lines.append("} // namespace functions")
     return "\n".join(lines)
 
 
+def _tools_sent_as_functions(tools: Sequence[object]) -> tuple[Mapping[str, object], ...]:
+    return tuple(tool for tool in _tool_mappings(tools) if not _is_anthropic_web_search_tool(tool))
+
+
+def _is_anthropic_web_search_tool(tool: Mapping[str, object]) -> bool:
+    return tool.get("name") == "web_search" and "input_schema" not in tool and tool.get("type") != "function"
+
+
+def _tool_mappings(tools: Sequence[object]) -> Iterable[Mapping[str, object]]:
+    for tool in tools:
+        if isinstance(tool, Mapping):
+            yield cast(Mapping[str, object], tool)
+
+
+def _function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]:
+    for tool in _tool_mappings(tools):
+        yield from _function_definitions_for_tool(tool)
+
+
 def _function_definitions_for_tool(tool: Mapping[str, object]) -> Iterable[Mapping[str, object]]:
     function: Final = tool.get("function")
     if isinstance(function, Mapping):
@@ -1054,12 +1112,21 @@ def _function_definitions_for_tool(tool: Mapping[str, object]) -> Iterable[Mappi
             if isinstance(declaration, Mapping):
                 yield declaration
         return
-    parameters: Final = tool.get("input_schema") or tool.get("parameters") or {}
+    input_schema: Final = tool.get("input_schema")
+    parameters: Final = input_schema or tool.get("parameters") or {}
     normalized_parameters: Final = parameters if isinstance(parameters, Mapping) else {}
+    unmapped_anthropic_keys: Final = (
+        {key: value for key, value in tool.items() if key not in _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE}
+        if input_schema is not None
+        else {}
+    )
+    description: Final = tool.get("description")
+    strict: Final = tool.get("strict")
     yield {
         "name": tool.get("name"),
-        "description": tool.get("description"),
-        "parameters": normalized_parameters,
+        "parameters": {**normalized_parameters, **unmapped_anthropic_keys},
+        **({"description": description} if description is not None else {}),
+        **({"strict": bool(strict)} if strict is not None else {}),
     }
 
 

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -30,6 +30,10 @@ from litellm.llms.anthropic.pass_through.adapters.transformation import (
     LiteLLMAnthropicMessagesAdapter,
     is_provider_native_tool_dict,
 )
+from litellm.llms.anthropic.pass_through.convertible_blocks import (
+    keep_non_string_tool_result_texts,
+    scannable_tool_result_request,
+)
 from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details
 from litellm.llms.anthropic.pass_through.stream_assembly import (
     build_complete_streaming_response,
@@ -493,8 +497,8 @@ class AnthropicMessagesHandler(BaseTranslation):
         ``get_streaming_string_so_far``): raw SSE ``bytes`` -- which may bundle
         several events separated by a blank line -- and an already-parsed event
         ``dict``."""
-        if isinstance(item, dict):
-            return (_as_str_mapping(item),)
+        if (event := as_str_mapping(item)) is not None:
+            return (event,)
         if not isinstance(item, (bytes, bytearray)):
             return ()
         return tuple(
@@ -510,7 +514,7 @@ class AnthropicMessagesHandler(BaseTranslation):
             chat_completion_compatible_request,
             _tool_name_mapping,
         ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
-            anthropic_message_request=cast(AnthropicMessagesRequest, data.copy()),
+            anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)),
             preserve_midturn_system=True,
         )
         return chat_completion_compatible_request
@@ -832,7 +836,7 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     @staticmethod
     def _write_back_structured_messages(
-        data: dict,  # mutable-ok: API message payload
+        data: dict[str, object],  # mutable-ok: API message payload
         structured_messages: list,  # mutable-ok: API message payload
         hoisted_system_message: object = None,
         preserve_system_messages: bool = False,
@@ -899,7 +903,7 @@ class AnthropicMessagesHandler(BaseTranslation):
                 for block in content:
                     if isinstance(block, dict) and block.get("type") == "thinking":
                         block.pop("cache_control", None)
-        data["messages"] = converted
+        data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages"))
 
     @staticmethod
     def _extract_midturn_system_text(
@@ -1601,8 +1605,8 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     @staticmethod
     def _rewrite_buffered_item(item: object, rewrite_event: _SSEEventRewriter) -> object:
-        if isinstance(item, dict):
-            return _rewritten_event(_as_str_mapping(item), rewrite_event)
+        if (event := as_str_mapping(item)) is not None:
+            return _rewritten_event(event, rewrite_event)
         if isinstance(item, (bytes, bytearray)):
             return AnthropicMessagesHandler._rewrite_sse_events(bytes(item), rewrite_event)
         return item

litellm/llms/anthropic/pass_through/adapters/handler.py

@@ -22,6 +22,7 @@ from litellm.llms.anthropic.pass_through.context_management import (
     PolyfillResult,
     apply_context_management,
 )
+from litellm.llms.anthropic.pass_through.convertible_blocks import reject_malformed_tool_result_text
 from litellm.llms.anthropic.pass_through.safeguards import (
     SafeguardsEvaluator,
     build_safeguards_evaluator,
@@ -731,6 +732,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
 
         proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
         request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs)
+        reject_malformed_tool_result_text(messages, model, request_kwargs)
         sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs)
         sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs)
 
@@ -853,6 +855,8 @@ class LiteLLMMessagesToCompletionTransformationHandler:
                 **kwargs,
             )
 
+        reject_malformed_tool_result_text(messages, model, _REQUEST_KWARGS.validate_python(kwargs))
+
         # Run the context_management polyfill on the sync path too so that
         # ``litellm.messages.create()`` callers don't silently lose edits like
         # ``clear_tool_uses_20250919``. The dispatcher is async (so the

litellm/llms/anthropic/pass_through/adapters/transformation.py

@@ -9,6 +9,7 @@ from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm.litellm_core_utils.hidden_params import get_hidden_params
+from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part
 from litellm.llms.anthropic.pass_through.utils import (
     is_reasoning_auto_summary_enabled,
     json_copy_or_original,
@@ -1334,7 +1335,8 @@ class LiteLLMAnthropicMessagesAdapter:
         block: Final = cast(Mapping[str, object], item)  # cast-ok: untrusted client payload
         match block.get("type"):
             case "text":
-                return ChatCompletionTextObject(type="text", text=str(block.get("text") or ""))
+                text: Final = tool_result_text_part(block)
+                return None if text is None else ChatCompletionTextObject(type="text", text=text)
             case "image" | "document":
                 return self._tool_result_image_part(block.get("source"))
             case "tool_reference":

litellm/llms/anthropic/pass_through/convertible_blocks.py

@@ -1,11 +1,18 @@
-from collections.abc import Iterator, Mapping, Sequence
-from itertools import chain
+import json
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from functools import partial
+from itertools import chain, groupby
+from operator import itemgetter
 from types import MappingProxyType
-from typing import Final
+from typing import Final, TypeAlias
 
 from pydantic import JsonValue, TypeAdapter, ValidationError
 from typing_extensions import assert_never
 
+import litellm
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
+from litellm.litellm_core_utils.prompt_templates.factory import sanitize_anthropic_tool_use_id
 from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
 from litellm.types.utils import TokenCountMessagesFormat
 
@@ -104,3 +111,245 @@ def unconvertible_blocks_error(
             )
         case _:
             assert_never(messages_format)
+
+
+_OBJECTS: Final = TypeAdapter(tuple[object, ...])
+
+
+def _list_items(value: object) -> tuple[object, ...]:
+    return _OBJECTS.validate_python(value) if isinstance(value, list) else ()
+
+
+@dataclass(frozen=True, slots=True)
+class _KeptPart:
+    block: object
+    scanned_text: str
+
+
+_KeptParts: TypeAlias = Mapping[int, _KeptPart]
+_NO_KEPT_PARTS: Final[_KeptParts] = MappingProxyType({})
+
+
+def _scannable_text(text: object) -> str:
+    return json.dumps(text, default=str, ensure_ascii=False)
+
+
+def _non_str_text(block: object) -> _KeptPart | None:
+    match as_str_mapping(block):
+        case {"type": "text", "text": str()}:
+            return None
+        case {"type": "text", "text": text}:
+            return _KeptPart(block=block, scanned_text=_scannable_text(text))
+        case _:
+            return None
+
+
+def tool_result_text_part(block: object) -> str | None:
+    match block:
+        case {"type": "text", "text": str(text)}:
+            return text
+        case {"type": "text", "text": object()}:
+            return None
+        case {"type": "text"}:
+            return ""
+        case _:
+            return None
+
+
+def tool_result_text_parts(tool_result: Mapping[str, object]) -> tuple[str, ...]:
+    parts: Final = map(tool_result_text_part, _list_items(tool_result.get("content")))
+    return tuple(part for part in parts if part is not None)
+
+
+def _user_content_blocks(message: object) -> tuple[object, ...]:
+    match as_str_mapping(message):
+        case {"role": "user", "content": content}:
+            return _list_items(content)
+        case _:
+            return ()
+
+
+def _with_user_blocks(message: object, transform: Callable[[object], object]) -> object:
+    match as_str_mapping(message):
+        case {"role": "user", "content": list()} as user_message:
+            return {**user_message, "content": [transform(block) for block in _user_content_blocks(user_message)]}
+        case _:
+            return message
+
+
+def _non_string_text_errors(label: str, content: object) -> Iterator[str]:
+    return (
+        f"{label} content.{index}.text: Input should be a valid string"
+        for index, text_block in enumerate(_list_items(content))
+        if _non_str_text(text_block) is not None
+    )
+
+
+def _tool_result_text_errors(block: object) -> Iterator[str]:
+    match block:
+        case {"type": "tool_result", "tool_use_id": object() as tool_use_id, "content": object() as content}:
+            return _non_string_text_errors(f"tool_result {tool_use_id}", content)
+        case {"type": "tool_result", "content": object() as content}:
+            return _non_string_text_errors("tool_result", content)
+        case _:
+            return iter(())
+
+
+def malformed_tool_result_text_error(messages: Iterable[object]) -> str | None:
+    blocks: Final = chain.from_iterable(map(_user_content_blocks, messages))
+    return next(chain.from_iterable(map(_tool_result_text_errors, blocks)), None)
+
+
+def malformed_tool_result_count_error(
+    messages: Iterable[Mapping[str, object]] | None,
+    messages_format: TokenCountMessagesFormat,
+    deployment: Mapping[str, object] | None,
+) -> str | None:
+    if messages_format != "anthropic" or deployment_drops_params(deployment):
+        return None
+    return malformed_tool_result_text_error(messages or ())
+
+
+def _scannable_part(part: object) -> object:
+    match as_str_mapping(part):
+        case {"type": "text", "text": str()}:
+            return part
+        case {"type": "text", "text": text} as text_block:
+            return {**text_block, "text": _scannable_text(text)}
+        case _:
+            return part
+
+
+def _scannable_block(block: object) -> object:
+    match as_str_mapping(block):
+        case {"type": "tool_result", "content": list()} as tool_result:
+            return {
+                **tool_result,
+                "content": [_scannable_part(part) for part in _list_items(tool_result.get("content"))],
+            }
+        case _:
+            return block
+
+
+def _scannable_messages(messages: object) -> object:
+    items: Final = _list_items(messages)
+    return [_with_user_blocks(message, _scannable_block) for message in items] if items else messages
+
+
+def scannable_tool_result_request(request: object) -> object:
+    match as_str_mapping(request):
+        case {"messages": messages} as request_fields:
+            return {**request_fields, "messages": _scannable_messages(messages)}
+        case None:
+            return request
+        case request_fields:
+            return {**request_fields}
+
+
+def _non_string_text_parts(content: object) -> _KeptParts:
+    return MappingProxyType(
+        {index: part for index, block in enumerate(_list_items(content)) if (part := _non_str_text(block)) is not None}
+    )
+
+
+def _tool_result_non_string_texts(block: object) -> tuple[tuple[str, _KeptParts], ...]:
+    match as_str_mapping(block):
+        case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result:
+            return ((sanitize_anthropic_tool_use_id(tool_use_id), _non_string_text_parts(tool_result.get("content"))),)
+        case _:
+            return ()
+
+
+def _content_blocks(content: object) -> tuple[object, ...]:
+    match content:
+        case str() if content:
+            return ({"type": "text", "text": content},)
+        case _:
+            return _list_items(content)
+
+
+def _restored_part(block: object, kept: _KeptPart | None) -> object:
+    return kept.block if kept is not None and tool_result_text_part(block) == kept.scanned_text else block
+
+
+def _with_kept_parts(block: object, tool_result: Mapping[str, object], parts: _KeptParts) -> object:
+    if not parts:
+        return block
+    blocks: Final = _content_blocks(tool_result.get("content"))
+    return {**tool_result, "content": [_restored_part(part, parts.get(index)) for index, part in enumerate(blocks)]}
+
+
+def _restored_block(block: object, kept: Mapping[str, Iterator[_KeptParts]]) -> object:
+    match as_str_mapping(block):
+        case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result if (
+            sanitize_anthropic_tool_use_id(tool_use_id) in kept
+        ):
+            return _with_kept_parts(
+                block, tool_result, next(kept[sanitize_anthropic_tool_use_id(tool_use_id)], _NO_KEPT_PARTS)
+            )
+        case _:
+            return block
+
+
+def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: object) -> Sequence[object]:
+    blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original)))
+    by_id: Final = tuple(sorted(chain.from_iterable(map(_tool_result_non_string_texts, blocks)), key=itemgetter(0)))
+    if not any(parts for _, parts in by_id):
+        return rewritten
+    kept_in_order: Final = MappingProxyType(
+        {
+            tool_use_id: iter(tuple(map(itemgetter(1), group)))
+            for tool_use_id, group in groupby(by_id, key=itemgetter(0))
+        }
+    )
+    return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten]
+
+
+def _without_non_string_text(block: object) -> object:
+    match as_str_mapping(block):
+        case {"type": "tool_result", "content": list()} as tool_result:
+            parts: Final = _list_items(tool_result.get("content"))
+            return {**tool_result, "content": [part for part in parts if _non_str_text(part) is None]}
+        case _:
+            return block
+
+
+def _without_non_string_texts(
+    message: dict[str, object],  # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
+) -> dict[str, object]:  # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
+    match message:
+        case {"role": "user", "content": list()}:
+            return {**message, "content": [_without_non_string_text(block) for block in _user_content_blocks(message)]}
+        case _:
+            return message
+
+
+def drop_non_string_tool_result_texts(
+    messages: Sequence[dict[str, object]],  # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
+    drop_params: object,
+) -> Sequence[dict[str, object]]:  # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
+    if not drops_unsupported_params(drop_params) or malformed_tool_result_text_error(messages) is None:
+        return messages
+    return [_without_non_string_texts(message) for message in messages]
+
+
+def deployment_drops_params(deployment: Mapping[str, object] | None) -> bool:
+    match deployment:
+        case {"litellm_params": {"drop_params": object() as drop_params}}:
+            return drops_unsupported_params(drop_params)
+        case _:
+            return drops_unsupported_params(None)
+
+
+def reject_malformed_tool_result_text(
+    messages: Iterable[Mapping[str, object]], model: str, forwarded_kwargs: Mapping[str, object]
+) -> None:
+    if drops_unsupported_params(forwarded_kwargs.get("drop_params")):
+        return
+    error: Final = malformed_tool_result_text_error(messages)
+    if error is None:
+        return
+    provider: Final = forwarded_kwargs.get("custom_llm_provider")
+    raise litellm.BadRequestError(
+        message=error, model=model, llm_provider=provider if isinstance(provider, str) else ""
+    )

litellm/llms/anthropic/pass_through/messages/handler.py

@@ -7,13 +7,14 @@
 
 import asyncio
 import contextvars
-from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
+from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
 from functools import partial
 from typing import Final, cast
 
 from pydantic import TypeAdapter
 
 import litellm
+from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
 from litellm.litellm_core_utils.exception_mapping_utils import exception_type
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.anthropic.common_utils import (
@@ -39,8 +40,7 @@ from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler
 from ..safeguards import (
     SafeguardsEvaluator,
     native_route_classifies,
-    read_classifier_model_setting,
-    requested_dangerous_tool_use,
+    proxy_serves_safeguards,
     with_safeguard_results,
 )
 from .interceptors import get_messages_interceptors
@@ -106,11 +106,17 @@ def _should_route_to_responses_api(
 
 async def _with_native_safeguard_results(
     pending: Coroutine[object, object, AnthropicMessagesResponse | AsyncIterator[bytes]],
-    evaluator: SafeguardsEvaluator,
+    proxy_classifies: Callable[[], Awaitable[bool]],
+    build_evaluator: Callable[[], SafeguardsEvaluator | None],
 ) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
     from .streaming_iterator import AnthropicMessagesStreamingResponse, SafeguardedAnthropicSSEStream
 
     response: Final = await pending
+    if not await proxy_classifies():
+        return response
+    evaluator: Final = await asyncio.to_thread(build_evaluator)
+    if evaluator is None:
+        return response
     if isinstance(response, AnthropicMessagesStreamingResponse):
         return AnthropicMessagesStreamingResponse(
             completion_stream=SafeguardedAnthropicSSEStream(response, evaluator),
@@ -121,6 +127,10 @@ async def _with_native_safeguard_results(
     return SafeguardedAnthropicSSEStream(response, evaluator)
 
 
+def _native_safeguards_evaluator(kwargs: Mapping[str, object], messages: object) -> SafeguardsEvaluator | None:
+    return request_safeguards_evaluator(kwargs, _SAFEGUARD_MESSAGES.validate_python(messages))
+
+
 def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
     """Whether the deployment opted into forwarding /v1/messages untranslated.
 
@@ -139,7 +149,7 @@ def _deployment_supports_cache_control_ttl(model_info: object) -> bool:
 
 
 def _native_messages_config(
-    model: str, custom_llm_provider: str | None, model_info: object
+    model: str, custom_llm_provider: str | None, model_info: object, api_base: str | None
 ) -> BaseAnthropicMessagesConfig | None:
     provider_config: Final = (
         ProviderConfigManager.get_provider_anthropic_messages_config(
@@ -149,8 +159,11 @@ def _native_messages_config(
         if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]
         else None
     )
-    if provider_config is not None or not _deployment_passes_through_anthropic_messages(model_info):
-        return provider_config
+    native_config: Final = (
+        provider_config if provider_config is not None and provider_config.serves_api_base(model, api_base) else None
+    )
+    if native_config is not None or not _deployment_passes_through_anthropic_messages(model_info):
+        return native_config
     from litellm.llms.openai_like.messages.transformation import (
         OpenAILikeAnthropicMessagesConfig,
     )
@@ -158,12 +171,26 @@ def _native_messages_config(
     return OpenAILikeAnthropicMessagesConfig(cache_control_ttl=_deployment_supports_cache_control_ttl(model_info))
 
 
-def proxy_classifies_tool_uses(model: str, custom_llm_provider: str | None, kwargs: Mapping[str, object]) -> bool:
+async def _proxy_classifies(model: str, safeguards: object, native_config: BaseAnthropicMessagesConfig | None) -> bool:
+    if not proxy_serves_safeguards(safeguards):
+        return False
+    if native_config is None:
+        return True
+    if native_config.should_filter_anthropic_beta_headers():
+        await aload_beta_headers_config()
+    return not native_config.supports_safeguards() or native_route_classifies(model, safeguards)
+
+
+async def proxy_classifies_tool_uses(model: str, custom_llm_provider: str | None, kwargs: Mapping[str, object]) -> bool:
     """Whether this proxy, rather than the upstream, answers ``safeguards`` for a request routed this way."""
-    safeguards: Final = kwargs.get("safeguards")
-    if _native_messages_config(model, custom_llm_provider, kwargs.get("model_info")) is not None:
-        return native_route_classifies(model, safeguards)
-    return requested_dangerous_tool_use(safeguards) is not None and read_classifier_model_setting() is not None
+    api_base: Final = kwargs.get("api_base")
+    return await _proxy_classifies(
+        model,
+        kwargs.get("safeguards"),
+        _native_messages_config(
+            model, custom_llm_provider, kwargs.get("model_info"), api_base if isinstance(api_base, str) else None
+        ),
+    )
 
 
 ####### ENVIRONMENT VARIABLES ###################
@@ -608,7 +635,7 @@ def anthropic_messages_handler(
             )
 
     anthropic_messages_provider_config: Final = _native_messages_config(
-        model, custom_llm_provider, kwargs.get("model_info")
+        model, custom_llm_provider, kwargs.get("model_info"), api_base
     )
     if anthropic_messages_provider_config is None:
         # Route to Responses API for OpenAI / Azure, chat/completions for everything else.
@@ -691,11 +718,6 @@ def anthropic_messages_handler(
     safeguard_kwargs: Final = (
         _SAFEGUARD_KWARGS.validate_python(kwargs) if is_async is True and "safeguards" in kwargs else None
     )
-    safeguards_evaluator: Final = (
-        request_safeguards_evaluator(safeguard_kwargs, _SAFEGUARD_MESSAGES.validate_python(messages))
-        if safeguard_kwargs is not None and native_route_classifies(model, safeguard_kwargs.get("safeguards"))
-        else None
-    )
     native_response: Final = base_llm_http_handler.anthropic_messages_handler(
         model=model,
         messages=cast(  # cast-ok: anthropic messages are dicts at runtime
@@ -716,6 +738,10 @@ def anthropic_messages_handler(
         stream=stream,
         kwargs=kwargs,
     )
-    if safeguards_evaluator is None or not isinstance(native_response, Coroutine):
+    if safeguard_kwargs is None or not isinstance(native_response, Coroutine):
         return native_response
-    return _with_native_safeguard_results(native_response, safeguards_evaluator)
+    return _with_native_safeguard_results(
+        native_response,
+        partial(_proxy_classifies, model, safeguard_kwargs.get("safeguards"), anthropic_messages_provider_config),
+        partial(_native_safeguards_evaluator, safeguard_kwargs, messages),
+    )

litellm/llms/anthropic/pass_through/messages/mcp_handler.py

@@ -136,7 +136,7 @@ async def anthropic_messages_with_mcp(
         )
 
     custom_llm_provider: Final = kwargs.get("custom_llm_provider")
-    classified_here: Final = proxy_classifies_tool_uses(
+    classified_here: Final = await proxy_classifies_tool_uses(
         model, custom_llm_provider if isinstance(custom_llm_provider, str) else None, kwargs
     )
     loop_call_args: Final[Mapping[str, object]] = (

litellm/llms/anthropic/pass_through/messages/transformation.py

@@ -3,10 +3,13 @@ from typing import ClassVar, Final
 
 import httpx
 
+import litellm
 from litellm._logging import verbose_logger
-from litellm.anthropic_beta_headers_manager import is_beta_header_supported_by_bundled_config
+from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config
 from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts
+from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
 )
@@ -110,6 +113,27 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     def should_filter_anthropic_beta_headers(self) -> bool:
         return self._resolved_provider != "anthropic"
 
+    def supports_safeguards(self) -> bool:
+        """
+        Whether the upstream platform answers the ``safeguards`` body field itself. A platform
+        whose beta header map column marks the dangerous-tool-use beta unsupported does not, and
+        a platform the map does not list is trusted to unless its config says otherwise.
+        """
+        return not self.should_filter_anthropic_beta_headers() or (
+            ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value
+            not in get_unsupported_headers(self._resolved_provider)
+        )
+
+    def drops_safeguards(self, safeguards: object, drop_params: bool) -> bool:
+        """
+        ``safeguards`` and the dangerous-tool-use beta leave together for a platform that does not
+        answer them, when the proxy classifies in its place or drop_params is set. Otherwise both
+        are forwarded as sent and the platform answers its own 400, like any unsupported param.
+        """
+        return not self.supports_safeguards() and (
+            drop_params or litellm.drop_params is True or proxy_serves_safeguards(safeguards)
+        )
+
     def _remove_scope_from_cache_control(self, anthropic_messages_request: dict) -> None:
         """
         Remove `scope` field from cache_control blocks.
@@ -208,7 +232,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         ids were found without tool_result blocks immediately after") while
         consecutive user turns merge upstream.
         Billing-header system blocks are stripped from the top-level ``system``
-        field regardless of whether anything was hoisted.
+        field when ``should_strip_billing_metadata`` says so, regardless of
+        whether anything was hoisted.
 
         Subclasses whose upstream rejects the role opt in by calling this from
         their ``transform_anthropic_messages_request``; the first-party Anthropic
@@ -242,7 +267,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             )
             for block in as_system_content_blocks(source)
         ]
-        filtered_system: Final = self._filter_billing_headers_from_system(system_content)
+        filtered_system: Final = (
+            self._filter_billing_headers_from_system(system_content)
+            if self.should_strip_billing_metadata()
+            else system_content
+        )
         if filtered_system:
             anthropic_messages_request["system"] = filtered_system
         else:
@@ -592,6 +621,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
                 status_code=400,
             )
 
+        if "safeguards" in anthropic_messages_optional_request_params and self.drops_safeguards(
+            anthropic_messages_optional_request_params["safeguards"],  # pyright: ignore[reportUnknownArgumentType]  # the request params arrive as a bare dict
+            litellm_params.drop_params is True,
+        ):
+            anthropic_messages_optional_request_params.pop("safeguards")  # pyright: ignore[reportUnknownMemberType]  # the request params arrive as a bare dict
+
         self._translate_reasoning_effort_to_anthropic(
             model=model,
             optional_params=anthropic_messages_optional_request_params,
@@ -658,7 +693,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             messages = strip_advisor_blocks_from_messages(messages)
 
         anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest(
-            messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(messages),
+            messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(
+                drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None))
+            ),
             max_tokens=max_tokens,
             model=model,
             **anthropic_messages_optional_request_params,
@@ -684,9 +721,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         self,
         model: str,
         httpx_response: httpx.Response,
-        request_body: dict,
+        request_body: dict[str, object],  # mutable-ok: mirrors the base signature
         litellm_logging_obj: LiteLLMLoggingObj,
-    ) -> AsyncIterator:
+    ) -> AsyncIterator[bytes]:
         """Helper function to handle Anthropic streaming responses using the existing logging handlers"""
         from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
             BaseAnthropicMessagesStreamingIterator,
@@ -703,13 +740,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             litellm_logging_obj=litellm_logging_obj,
         )
 
-    @staticmethod
     def _update_headers_with_anthropic_beta(
+        self,
         headers: dict,
         optional_params: dict,
         custom_llm_provider: str = "anthropic",
         messages: Sequence[object] = (),
         beta_map_provider: str | None = None,
+        drop_params: bool = False,
     ) -> dict:
         """
         Auto-inject anthropic-beta headers based on features used.
@@ -727,6 +765,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             custom_llm_provider: Provider name for looking up correct tool search header
             messages: Request messages, scanned for per-message output_config
             beta_map_provider: Provider whose beta map column gates safeguards, defaults to custom_llm_provider
+            drop_params: Whether the deployment or request set drop_params
         """
         beta_values: Final[set[str]] = set()
 
@@ -805,9 +844,20 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             else ()
         )
         all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas)
+        dangerous_tool_use: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value
+        forwarded_beta_values: Final = (  # pyright: ignore[reportUnknownVariableType]  # derived from the bare beta set
+            all_beta_values - {dangerous_tool_use}
+            if dangerous_tool_use in all_beta_values
+            and self.drops_safeguards(
+                optional_params.get("safeguards"),  # pyright: ignore[reportUnknownArgumentType]  # the optional params arrive as a bare dict
+                drop_params,
+            )
+            else all_beta_values
+        )
 
         if not all_beta_values:
             return headers
         merged: Final = {key: value for key, value in headers.items() if key.lower() != "anthropic-beta"}
-        merged["anthropic-beta"] = ",".join(sorted(all_beta_values))
+        if forwarded_beta_values:
+            merged["anthropic-beta"] = ",".join(sorted(forwarded_beta_values))
         return merged

litellm/llms/anthropic/pass_through/responses_adapters/handler.py

@@ -14,7 +14,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
     AnthropicCacheControlHook,
     supports_openai_prompt_cache_breakpoint,
 )
-from litellm.litellm_core_utils.core_helpers import normalize_drop_params
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
 from litellm.responses.utils import auto_reasoning_summary
 from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
@@ -32,6 +32,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse
 from litellm.utils import ProviderConfigManager
 
 from ..adapters.handler import request_safeguards_evaluator
+from ..convertible_blocks import reject_malformed_tool_result_text
 from ..safeguards import with_safeguard_results
 from ..utils import litellm_logging_obj_from_kwargs, local_model_name
 from .streaming_iterator import AnthropicResponsesStreamWrapper
@@ -96,7 +97,7 @@ def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, objec
         return cache_control_as_prompt_cache_breakpoint
     if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs):
         return None
-    if normalize_drop_params(forwarded_kwargs.get("drop_params")) is True or litellm.drop_params is True:
+    if drops_unsupported_params(forwarded_kwargs.get("drop_params")):
         return _without_prompt_cache_breakpoint
     return None
 
@@ -142,6 +143,7 @@ def _build_responses_kwargs(
     Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses().
     """
     forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
+    reject_malformed_tool_result_text(messages, model, forwarded_kwargs)
     block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs)
     request_messages: Final[AnthropicRequestMessages] = (
         [

litellm/llms/anthropic/pass_through/responses_adapters/transformation.py

@@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking_budget,
 )
+from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_parts
 from litellm.llms.anthropic.pass_through.messages.utils import (
     refusal_stop_details,
     responses_output_refusal_text,
@@ -371,10 +372,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                         elif btype == "tool_result":
                             tool_use_id = block.get("tool_use_id", "")
                             inner = block.get("content")
+                            tool_result_block = cast(Mapping[str, object], block)  # cast-ok: TypedDict upcast
                             tool_breakpoint, covered_image_parts, later_image_parts = (
-                                self._tool_result_images_by_breakpoint(
-                                    cast(Mapping[str, object], block)  # cast-ok: TypedDict upcast
-                                )
+                                self._tool_result_images_by_breakpoint(tool_result_block)
                             )
                             document_candidates = (
                                 tuple(
@@ -391,10 +391,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                             elif isinstance(inner, str):
                                 output_text = inner
                             elif isinstance(inner, list):
-                                parts = [
-                                    c.get("text", "") for c in inner if isinstance(c, dict) and c.get("type") == "text"
-                                ]
-                                output_text = "\n".join(parts)
+                                output_text = "\n".join(tool_result_text_parts(tool_result_block))
                                 if covered_image_parts or later_image_parts:
                                     output_text = (
                                         f"{output_text}\n{TOOL_RESULT_IMAGE_PLACEHOLDER}"

litellm/llms/anthropic/pass_through/safeguards.py

@@ -273,12 +273,12 @@ def read_classifier_model_setting() -> str | None:
     return configured if isinstance(configured, str) and configured else None
 
 
+def proxy_serves_safeguards(safeguards: object) -> bool:
+    return requested_dangerous_tool_use(safeguards) is not None and read_classifier_model_setting() is not None
+
+
 def native_route_classifies(model: str, safeguards: object) -> bool:
-    return (
-        "claude" not in model.lower()
-        and requested_dangerous_tool_use(safeguards) is not None
-        and read_classifier_model_setting() is not None
-    )
+    return "claude" not in model.lower() and proxy_serves_safeguards(safeguards)
 
 
 def build_safeguards_evaluator(

litellm/llms/azure/chat/gpt_5_transformation.py

@@ -1,12 +1,10 @@
 """Support for Azure OpenAI gpt-5 model family."""
 
-from typing import Final
+from typing import ClassVar, Final
 
 import litellm
-from litellm.exceptions import UnsupportedParamsError
 from litellm.llms.openai.chat.gpt_5_transformation import (
     OpenAIGPT5Config,
-    get_effort_level,
     is_gpt_reasoning_series_name,
 )
 from litellm.types.llms.openai import AllMessageValues
@@ -15,9 +13,16 @@ from .gpt_transformation import AzureOpenAIConfig
 
 
 class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
-    """Azure specific handling for gpt-5 models."""
+    """Azure specific handling for gpt-5 models.
+
+    reasoning_effort none joins xhigh as opt-in here: Azure rejected it on every gpt-5
+    deployment before gpt-5.1 (https://github.com/BerriAI/litellm/issues/16704), and a deployment
+    name the map never heard of must not forward it. See
+    https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning
+    """
 
     GPT5_SERIES_ROUTE = "gpt5_series/"
+    OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none"))
 
     @classmethod
     def _model_map_lookup_name(cls, model: str) -> str:
@@ -27,11 +32,18 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
         model_prices_and_context_window.json; ``azure/gpt-5.1`` is. Overriding the shared
         resolver rather than one lookup means the supports, explicitly-disabled and
         default-effort answers all read the same entry.
+
+        An ``azure_ai/`` name reaches this config when ``get_llm_provider`` re-routes a Foundry
+        deployment on an OpenAI-v1 host to the azure provider with the prefix intact. It is the
+        same product the azure_ai config reads ``azure_ai/<model>`` for, so that row answers when
+        the map carries it, and the azure/ twin when it does not.
         """
         if model.startswith(cls.GPT5_SERIES_ROUTE):
             return "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :]
         if model.startswith("azure/"):
             return model
+        if model.startswith("azure_ai/"):
+            return model if model in litellm.model_cost else "azure/" + model.removeprefix("azure_ai/")
         return "azure/" + model
 
     @classmethod
@@ -75,33 +87,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
         drop_params: bool,
         api_version: str = "",
     ) -> dict:
-        reasoning_effort_value = non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort")
-        effective_effort: Final = get_effort_level(reasoning_effort_value)
-
-        # gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't
-        # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning
-        supports_none: Final = self._supports_reasoning_effort_level(model, "none")
-
-        if effective_effort == "none" and not supports_none:
-            if litellm.drop_params is True or (drop_params is not None and drop_params is True):
-                non_default_params = non_default_params.copy()
-                optional_params = optional_params.copy()
-                if get_effort_level(non_default_params.get("reasoning_effort")) == "none":
-                    non_default_params.pop("reasoning_effort")
-                if get_effort_level(optional_params.get("reasoning_effort")) == "none":
-                    optional_params.pop("reasoning_effort")
-            else:
-                raise UnsupportedParamsError(
-                    status_code=400,
-                    message=(
-                        "Azure OpenAI does not support reasoning_effort='none' for this model. "
-                        "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n"
-                        "`litellm_settings:\n drop_params: true`\n"
-                        "Issue: https://github.com/BerriAI/litellm/issues/16704"
-                    ),
-                )
-
-        result: Final = OpenAIGPT5Config.map_openai_params(
+        return OpenAIGPT5Config.map_openai_params(
             self,
             non_default_params=non_default_params,
             optional_params=optional_params,
@@ -109,17 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
             drop_params=drop_params,
         )
 
-        # Only drop reasoning_effort='none' for models that don't support it
-        result_effort: Final = get_effort_level(result.get("reasoning_effort"))
-        if result_effort == "none" and not supports_none:
-            result.pop("reasoning_effort")
-
-        # Azure gpt-5.4+ with tools + reasoning_effort is now routed to the
-        # Responses API bridge (same as OpenAI), so we no longer need to drop
-        # reasoning_effort here.  See: responses_api_bridge_check() in main.py.
-
-        return result
-
     def transform_request(
         self,
         model: str,

litellm/llms/azure/common_utils.py

@@ -3,13 +3,15 @@ import hashlib
 import json
 import os
 import re
-from collections.abc import Callable, Mapping
+from collections.abc import Awaitable, Callable, Mapping
+from contextlib import closing
 from functools import lru_cache
 from types import MappingProxyType
 from typing import Final, Literal, NamedTuple, cast
 
 import httpx
 from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+from pydantic import ConfigDict, TypeAdapter, with_config
 from typing_extensions import ReadOnly, TypedDict
 
 import litellm
@@ -17,6 +19,7 @@ from litellm._logging import verbose_logger
 from litellm.caching.caching import DualCache
 from litellm.constants import DEFAULT_MAX_RETRIES
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.custom_httpx.http_handler import HTTPHandler
 from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient
 from litellm.secret_managers.get_azure_ad_token_provider import (
     get_azure_ad_token_provider,
@@ -38,6 +41,16 @@ class _AzureV1ClientParams(TypedDict, total=False, extra_items=object):
     base_url: ReadOnly[str]
 
 
+@with_config(ConfigDict(extra="allow"))
+class _AsyncAzureClientParams(TypedDict, total=False, extra_items=object):
+    api_key: ReadOnly[str | None]
+    azure_ad_token: ReadOnly[str | None]
+    azure_ad_token_provider: ReadOnly[Callable[[], str | Awaitable[str]] | None]
+
+
+_ASYNC_AZURE_CLIENT_PARAMS_ADAPTER: Final = TypeAdapter(_AsyncAzureClientParams)
+
+
 class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object):
     api_version: ReadOnly[str]
     base_url: ReadOnly[str]
@@ -98,11 +111,42 @@ def _cached_entra_id_token_provider(
     return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
 
 
+@lru_cache(maxsize=128)
+def _cached_username_password_token_provider(
+    client_id: str,
+    azure_username: str,
+    azure_password: str,
+    scope: str,
+) -> Callable[[], str]:
+    from azure.identity import UsernamePasswordCredential, get_bearer_token_provider
+
+    credential: Final = UsernamePasswordCredential(
+        client_id=client_id,
+        username=azure_username,
+        password=azure_password,
+    )
+    return get_bearer_token_provider(credential, scope)
+
+
 @lru_cache(maxsize=128)
 def _cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]:
     return get_azure_ad_token_provider(azure_scope=scope)
 
 
+def cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]:
+    return _cached_azure_ad_token_refresh_provider(scope)
+
+
+def _async_azure_ad_token_provider(
+    provider: Callable[[], str | Awaitable[str]],
+) -> Callable[[], Awaitable[str]]:
+    async def async_provider() -> str:
+        token: Final = await asyncio.to_thread(provider)
+        return token if isinstance(token, str) else await token
+
+    return async_provider
+
+
 def get_azure_ad_token_from_entra_id(
     tenant_id: str,
     client_id: str,
@@ -177,22 +221,19 @@ def get_azure_ad_token_from_username_password(
     Returns:
         callable that returns a bearer token.
     """
-    from azure.identity import UsernamePasswordCredential, get_bearer_token_provider
-
     verbose_logger.debug(
         "client_id=%s, azure_username=[set=%s], azure_password=[set=%s]",
         client_id,
         azure_username is not None,
         azure_password is not None,
     )
-    credential: Final = UsernamePasswordCredential(
+    token_provider: Final = _cached_username_password_token_provider(
         client_id=client_id,
-        username=azure_username,
-        password=azure_password,
+        azure_username=azure_username,
+        azure_password=azure_password,
+        scope=scope,
     )
 
-    token_provider: Final = get_bearer_token_provider(credential, scope)
-
     verbose_logger.debug("token_provider %s", token_provider)
 
     return token_provider
@@ -248,18 +289,17 @@ def get_azure_ad_token_from_oidc(
     if azure_ad_token_access_token is not None:
         return azure_ad_token_access_token
 
-    client: Final = litellm.module_level_client
-
-    req_token: Final = client.post(
-        f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token",
-        data={
-            "client_id": azure_client_id,
-            "grant_type": "client_credentials",
-            "scope": scope,
-            "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
-            "client_assertion": oidc_token,
-        },
-    )
+    with closing(HTTPHandler(timeout=litellm.request_timeout)) as client:
+        req_token: Final = client.post(
+            f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token",
+            data={
+                "client_id": azure_client_id,
+                "grant_type": "client_credentials",
+                "scope": scope,
+                "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
+                "client_assertion": oidc_token,
+            },
+        )
 
     if req_token.status_code != 200:
         raise AzureOpenAIError(
@@ -375,7 +415,7 @@ def get_azure_ad_token(
             "Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth"
         )
         try:
-            azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope)
+            azure_ad_token_provider = _cached_azure_ad_token_refresh_provider(scope)
         except ValueError:
             verbose_logger.debug("Azure AD Token Provider could not be used.")
         except Exception as e:
@@ -560,22 +600,19 @@ class BaseAzureLLM(BaseOpenAILLM):
             # on every request (via `_refresh_api_key`), so passing
             # `azure_ad_token_provider` directly preserves Azure AD token refresh
             # behavior that the regular AzureOpenAI client provides.
-            v1_api_key: str | Callable[[], object] | None = (
-                azure_client_params.get("api_key")
-                or azure_client_params.get("azure_ad_token_provider")
-                or azure_client_params.get("azure_ad_token")
+            v1_client_params: Final[_AsyncAzureClientParams] = _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER.validate_python(
+                azure_client_params
+            )
+            v1_token_provider: Final = (
+                v1_client_params.get("api_key")
+                or v1_client_params.get("azure_ad_token_provider")
+                or v1_client_params.get("azure_ad_token")
+            )
+            v1_api_key: Final[str | Callable[[], str | Awaitable[str]] | None] = (
+                _async_azure_ad_token_provider(v1_token_provider)
+                if _is_async is True and callable(v1_token_provider)
+                else v1_token_provider
             )
-            if _is_async is True and callable(v1_api_key):
-                # AsyncOpenAI expects an async provider; wrap the sync provider
-                # returned by azure-identity. Offload to a thread so a token
-                # refresh (blocking HTTP call to AAD on cache miss) does not
-                # stall the event loop.
-                _sync_provider: Final = v1_api_key
-
-                async def _async_v1_api_key() -> str:
-                    return await asyncio.to_thread(_sync_provider)
-
-                v1_api_key = _async_v1_api_key
 
             v1_params: Final[_AzureV1ClientParams] = {
                 "api_key": v1_api_key,
@@ -597,7 +634,19 @@ class BaseAzureLLM(BaseOpenAILLM):
         else:
             # Traditional Azure API uses AzureOpenAI client
             if _is_async is True:
-                openai_client = AsyncAzureOpenAI(**azure_client_params)
+                async_azure_client_params: Final[_AsyncAzureClientParams] = (
+                    _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER.validate_python(azure_client_params)
+                )
+                async_token_provider_candidate: Final = async_azure_client_params.get("azure_ad_token_provider")
+                async_azure_client_params_with_threaded_provider: Final[_AsyncAzureClientParams] = (
+                    {
+                        **async_azure_client_params,
+                        "azure_ad_token_provider": _async_azure_ad_token_provider(async_token_provider_candidate),
+                    }
+                    if async_token_provider_candidate is not None
+                    else async_azure_client_params
+                )
+                openai_client = AsyncAzureOpenAI(**async_azure_client_params_with_threaded_provider)
             else:
                 openai_client = AzureOpenAI(**azure_client_params)
 

litellm/llms/azure/realtime/http_transformation.py

@@ -7,17 +7,21 @@ from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPC
 from litellm.secret_managers.main import get_secret_str
 
 
+def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str:
+    """Azure serves WebRTC only on the GA v1 surface, which rejects any api-version query param."""
+    base: Final = api_base.rstrip("/").removesuffix("/v1").removesuffix("/openai")
+    return f"{base}/openai/v1/realtime/{endpoint}"
+
+
 class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
-    def get_api_base(self, api_base: str | None, **kwargs) -> str:
+    def get_api_base(self, api_base: str | None, **kwargs: object) -> str:
         return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or ""
 
-    def get_api_key(self, api_key: str | None, **kwargs) -> str:
+    def get_api_key(self, api_key: str | None, **kwargs: object) -> str:
         return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or ""
 
     def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
-        base: Final = self.get_api_base(api_base).rstrip("/")
-        version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
-        return f"{base}/openai/realtime/client_secrets?api-version={version}"
+        return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets")
 
     def validate_environment(
         self,
@@ -32,16 +36,14 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
         }
 
     def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
-        base: Final = self.get_api_base(api_base).rstrip("/")
-        version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
-        return f"{base}/openai/realtime/calls?api-version={version}"
+        return _azure_realtime_v1_url(self.get_api_base(api_base), "calls")
 
     def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
         base: Final = self.get_api_base(api_base).rstrip("/")
         version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
         return f"{base}/openai/realtime/transcription_sessions?api-version={version}"
 
-    def get_realtime_calls_headers(self, ephemeral_key: str) -> dict:
+    def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]:
         return {
-            "api-key": ephemeral_key,
+            "Authorization": f"Bearer {ephemeral_key}",
         }

litellm/llms/azure/responses/transformation.py

@@ -65,10 +65,18 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def custom_llm_provider(self) -> LlmProviders:
         return LlmProviders.AZURE
 
+    @staticmethod
+    def _is_gpt_5_model(model: str) -> bool:
+        return AzureOpenAIGPT5Config.is_model_gpt_5_model(model)
+
     @staticmethod
     def _supports_reasoning_effort_none(model: str) -> bool:
         return AzureOpenAIGPT5Config.supports_reasoning_effort_level(model, "none")
 
+    @staticmethod
+    def _effort_level_is_disabled(model: str, level: str) -> bool:
+        return AzureOpenAIGPT5Config.row_disables_reasoning_effort_level(model, level)
+
     @staticmethod
     def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
         return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)

litellm/llms/azure/search/transformation.py

@@ -35,6 +35,7 @@ from typing import TYPE_CHECKING, Final, Literal
 import httpx
 from pydantic import ConfigDict, ValidationError
 
+from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.search.transformation import (
     BaseSearchConfig,
@@ -263,9 +264,7 @@ def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool |
 
 
 def _default_entra_token_minter() -> str:
-    from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider
-
-    return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)()
+    return cached_azure_ad_token_refresh_provider(ENTRA_SCOPE)()
 
 
 class BingGroundingSearchConfig(BaseSearchConfig):

litellm/llms/azure_ai/responses/transformation.py

@@ -3,6 +3,7 @@ from typing import Final
 import httpx
 
 from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
+from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config
 from litellm.llms.azure_ai.common_utils import (
     AzureFoundryModelInfo,
     api_key_header_for_base,
@@ -27,6 +28,18 @@ class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig):
     def custom_llm_provider(self) -> LlmProviders:
         return LlmProviders.AZURE_AI
 
+    @staticmethod
+    def _supports_reasoning_effort_none(model: str) -> bool:
+        return AzureAIGPT5Config.supports_reasoning_effort_level(model, "none")
+
+    @staticmethod
+    def _effort_level_is_disabled(model: str, level: str) -> bool:
+        return AzureAIGPT5Config.row_disables_reasoning_effort_level(model, level)
+
+    @staticmethod
+    def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
+        return AzureAIGPT5Config.effort_resolves_to_none(model, effort)
+
     def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict:
         params: Final = litellm_params or GenericLiteLLMParams()
         auth_headers: Final = get_azure_ai_auth_headers(