Merge origin/main into litellm_prometheus_provider_401_failure_metrics

Resolve the prometheus.py import conflict with #47384 and keep one _logged_fields helper, which returns an empty mapping for any value that is not a str-keyed mapping.

.github/e2e-stack/assert_tests_ran.py

@@ -1,69 +0,0 @@
-import os
-import re
-import sys
-import xml.etree.ElementTree as ET
-from pathlib import Path
-from typing import Final
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests/e2e"))
-from coverage_registry.management_cases import MANAGEMENT_CASES
-
-
-def main() -> int:
-    selected: Final = tuple(sys.argv[2:])
-    try:
-        report: Final = ET.parse(Path(sys.argv[1])).getroot()
-    except (ET.ParseError, OSError):
-        _ = sys.stdout.write("::error::could not read the test execution report\n")
-        return 1
-    cases: Final = tuple(report.iter("testcase"))
-    expected_count: Final = os.environ.get("E2E_REQUIRED_TEST_COUNT")
-    passed: Final = frozenset(
-        case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
-    )
-    missing: Final = tuple(path for path in selected if path not in passed)
-    required_nodes: Final = frozenset(case.node for case in MANAGEMENT_CASES if case.node.split("::", 1)[0] in selected)
-    passed_nodes: Final = frozenset(
-        prop.get("value")
-        for case in cases
-        if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
-        for prop in case.findall("./properties/property")
-        if prop.get("name") == "management_node"
-    )
-    missing_nodes: Final = required_nodes - passed_nodes
-    for node in sorted(missing_nodes):
-        _ = sys.stdout.write(f"::error::required management case did not pass: {node}\n")
-    for path in selected:
-        collected: Final = sum(case.get("file") == path for case in cases)
-        skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
-        _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
-        for case in cases:
-            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error", "skipped")):
-                continue
-            outcome = "skipped" if case.find("skipped") is not None else "failed"
-            _ = sys.stdout.write(f"  {outcome}: {case.get('classname', '')}::{case.get('name', '')}\n")
-            for prop in case.findall("./properties/property"):
-                name = prop.get("name", "")
-                value = prop.get("value", "")
-                if name in ("oauth_failure_phase", "oauth_exception_type", "oauth_frame") and re.fullmatch(
-                    r"[A-Za-z0-9_.:<>-]{1,240}", value
-                ):
-                    _ = sys.stdout.write(f"    {name}: {value}\n")
-    if expected_count is not None and (
-        len(cases) != int(expected_count) or any(case.find("skipped") is not None for case in cases)
-    ):
-        _ = sys.stdout.write("::error::required test count was not met or a required case was skipped\n")
-        return 1
-    if (
-        selected
-        and not missing
-        and not missing_nodes
-        and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
-    ):
-        return 0
-    _ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
-    return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/workflows/test-linting.yml

@@ -539,8 +539,8 @@ jobs:
       - name: test_unit_passed_gate
         run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py
 
-      - name: test_e2e_changed_gate
-        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
+      - name: test_e2e_idp_stack
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_idp_stack.py
 
       - name: test_e2e_metadata
         env:

.github/workflows/test-mcp-oauth-e2e.yml

@@ -1,180 +0,0 @@
-name: MCP OAuth happy path
-
-on:
-  pull_request:
-    paths:
-      - '.github/workflows/test-mcp-oauth-e2e.yml'
-      - '.github/e2e-stack/**'
-      - 'tests/e2e/*.py'
-      - 'tests/e2e/pytest.ini'
-      - 'tests/e2e/idp_realm.json'
-      - 'tests/e2e/mcp/**'
-      - 'litellm/experimental_mcp_client/**'
-      - 'litellm/proxy/_experimental/mcp_server/**'
-      - 'litellm/proxy/auth/**'
-      - 'litellm/proxy/management_endpoints/mcp_management_endpoints.py'
-      - 'litellm/proxy/_types.py'
-      - 'litellm/types/mcp_server/mcp_server_manager.py'
-      - 'litellm/proxy/management_endpoints/*sso*.py'
-      - 'litellm/proxy/management_endpoints/sso/**'
-      - 'litellm/proxy/common_utils/encrypt_decrypt_utils.py'
-      - 'litellm/proxy/proxy_server.py'
-      - 'litellm/proxy/schema.prisma'
-      - 'ui/litellm-dashboard/src/app/connect/**'
-      - 'ui/litellm-dashboard/src/app/mcp/oauth/**'
-      - 'pyproject.toml'
-      - 'uv.lock'
-  workflow_dispatch:
-
-permissions: {}
-
-concurrency:
-  group: mcp-oauth-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  oauth:
-    if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
-    runs-on: ubuntu-latest
-    environment: e2e-changed
-    timeout-minutes: 45
-    permissions:
-      contents: read
-      id-token: write
-    services:
-      postgres:
-        image: postgres:16.6
-        env:
-          POSTGRES_USER: litellm
-          POSTGRES_PASSWORD: dbpassword9090
-          POSTGRES_DB: litellm
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd "pg_isready -U litellm"
-          --health-interval 5s
-          --health-timeout 5s
-          --health-retries 10
-    env:
-      DATABASE_HOST: 127.0.0.1
-      DATABASE_PORT: '5432'
-      DATABASE_USER: litellm
-      DATABASE_PASSWORD: dbpassword9090
-      DATABASE_NAME: litellm
-      DATABASE_URL: postgresql://litellm:dbpassword9090@127.0.0.1:5432/litellm
-      E2E_KEYCLOAK_URL: http://127.0.0.1:8081
-      E2E_KEYCLOAK_ADMIN_USER: admin
-      E2E_KEYCLOAK_ADMIN_PASSWORD: e2e-ephemeral-idp-not-a-secret
-      E2E_FIXTURE_MODE: live
-      E2E_PROVIDER_CACHE: '0'
-      E2E_MCP_OAUTH_LIVE: '1'
-      E2E_REQUIRED_TEST_COUNT: '4'
-    steps:
-      - name: Checkout the tested source
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.sha }}
-          persist-credentials: false
-
-      - name: Require and materialize the upstream login
-        env:
-          STORAGE_STATE: ${{ secrets.E2E_LINEAR_STORAGE_STATE_B64 }}
-        run: |
-          umask 077
-          python3 - <<'PY'
-          import base64
-          import json
-          import os
-          import secrets
-          from pathlib import Path
-          encoded = os.environ.get("STORAGE_STATE", "")
-          if not encoded:
-              raise SystemExit("E2E_LINEAR_STORAGE_STATE_B64 is required; capture and provision a test-account login")
-          state = json.loads(base64.b64decode(encoded, validate=True))
-          if not isinstance(state, dict) or not state.get("cookies"):
-              raise SystemExit("The captured login must contain browser cookies")
-          directory = Path(os.environ["RUNNER_TEMP"]) / "mcp-oauth-private"
-          directory.mkdir(mode=0o700)
-          path = directory / "linear-state.json"
-          path.write_text(json.dumps(state))
-          with open(os.environ["GITHUB_ENV"], "a") as output:
-              output.write(f"E2E_LINEAR_STORAGE_STATE={path}\n")
-              for name in ("LITELLM_MASTER_KEY", "LITELLM_SALT_KEY"):
-                  value = "sk-e2e-" + secrets.token_hex(24)
-                  print(f"::add-mask::{value}")
-                  output.write(f"{name}={value}\n")
-          PY
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: '3.13'
-      - uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: '0.10.9'
-      - uses: ./.github/actions/cache-cargo-build
-      - name: Install the frozen E2E environment
-        run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra proxy-runtime --extra extra_proxy --group ci --group proxy-dev --group e2e-dev
-          uv run --no-sync python scripts/prisma_generate_if_needed.py
-          uv run --no-sync playwright install --with-deps chromium
-
-      - name: Configure license access
-        id: aws
-        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
-        with:
-          role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
-          aws-region: us-east-1
-          role-session-name: mcp-oauth-${{ github.run_id }}
-          role-duration-seconds: 900
-          output-env-credentials: false
-          output-credentials: true
-      - name: Load the E2E license
-        env:
-          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
-          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
-          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
-          AWS_DEFAULT_REGION: us-east-1
-        run: |
-          license="$(aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license --query SecretString --output text)"
-          test -n "${license}"
-          echo "::add-mask::${license}"
-          echo "LITELLM_LICENSE=${license}" >> "${GITHUB_ENV}"
-
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
-        with:
-          node-version-file: ui/litellm-dashboard/.nvmrc
-      - name: Build the gateway consent UI at the tested commit
-        run: |
-          cd ui/litellm-dashboard
-          ../../scripts/with_dashboard_node.sh npm ci
-          ../../scripts/with_dashboard_node.sh npm run build
-          mkdir -p ../../litellm/proxy/_experimental/out
-          cp -r out/. ../../litellm/proxy/_experimental/out/
-          find ../../litellm/proxy/_experimental/out -name '*.html' ! -name index.html | while read -r page; do
-            mkdir -p "${page%.html}"
-            mv "${page}" "${page%.html}/index.html"
-          done
-
-      - name: Prepare the isolated database and IdP
-        run: |
-          umask 077
-          bash .github/e2e-stack/start-idp.sh
-          uv run --no-sync python migrations/run.py > "${RUNNER_TEMP}/mcp-oauth-private/migrations.log" 2>&1
-
-      - name: Run every required OAuth variant without retries
-        run: |
-          umask 077
-          uv run --no-sync pytest -c tests/e2e/pytest.ini tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py \
-            --rootdir=. --reruns 0 --tb=short -o junit_family=xunit1 \
-            --junitxml="${RUNNER_TEMP}/mcp-oauth-private/results.xml" \
-            > "${RUNNER_TEMP}/mcp-oauth-private/pytest.log" 2>&1
-      - name: Report JUnit results and reject skipped or missing cases
-        if: always()
-        run: |
-          uv run --no-sync python .github/e2e-stack/assert_tests_ran.py \
-            "${RUNNER_TEMP}/mcp-oauth-private/results.xml" tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py
-      - name: Remove private login and logs
-        if: always()
-        run: |
-          docker rm -f e2e-keycloak >/dev/null 2>&1 || true
-          rm -rf "${RUNNER_TEMP}/mcp-oauth-private"

.github/workflows/test-unit.yml

@@ -410,6 +410,7 @@ jobs:
               tests/unit/embeddings
               tests/unit/endpoints
               tests/unit/files
+              tests/unit/fine_tuning
               tests/unit/images
               tests/unit/interactions
               tests/unit/messages
@@ -658,6 +659,7 @@ jobs:
           - shard: proxy-extras
             test-path: >-
               tests/unit/litellm_proxy_extras
+              tests/unit/db_scripts
             python-version: "3.12"
             workers: 2
             reruns: 0

backend/routes/allowlist.py

@@ -165,6 +165,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
 BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
     {
         "/admin",
+        "/metrics",
         "/swagger",  # API documentation static assets belong to the backend
         "/mcp",  # lazily-mounted MCP sub-app serves on the backend component
     }

db_scripts/strip_vector_store_internal_metadata.py

@@ -0,0 +1,201 @@
+import argparse
+import os
+import sys
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from itertools import accumulate, repeat, takewhile
+from typing import Final, Literal, TextIO
+
+import httpx
+from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
+
+from litellm.vector_stores.utils import is_litellm_written_metadata_key
+
+DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
+LIST_PAGE_SIZE: Final = "100"
+
+
+class VectorStore(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    id: str
+    metadata: dict[str, JsonValue] | None = None
+
+
+class VectorStorePage(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    data: tuple[VectorStore, ...]
+    has_more: bool = False
+
+
+class OpenAIErrorBody(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    message: str = ""
+
+
+class OpenAIErrorResponse(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    error: OpenAIErrorBody
+
+
+@dataclass(frozen=True, slots=True)
+class Options:
+    api_key: str
+    api_base: str
+    vector_store_ids: tuple[str, ...]
+    extra_keys: frozenset[str]
+    apply: bool
+
+
+@dataclass(frozen=True, slots=True)
+class StoreResult:
+    store_id: str
+    matched_keys: tuple[str, ...]
+    status: Literal["dry_run", "updated", "failed"]
+    error: str = ""
+
+
+def is_proxy_written_key(key: str, extra_keys: frozenset[str]) -> bool:
+    return is_litellm_written_metadata_key(key) or key in extra_keys
+
+
+def matching_keys(store: VectorStore, extra_keys: frozenset[str]) -> tuple[str, ...]:
+    return tuple(sorted(key for key in (store.metadata or {}) if is_proxy_written_key(key, extra_keys)))
+
+
+def fetch_page(client: httpx.Client, after: str | None) -> VectorStorePage:
+    params: Final = {"limit": LIST_PAGE_SIZE} if after is None else {"limit": LIST_PAGE_SIZE, "after": after}
+    return VectorStorePage.model_validate_json(client.get("/vector_stores", params=params).raise_for_status().content)
+
+
+def fetch_next_page(client: httpx.Client, page: VectorStorePage | None) -> VectorStorePage | None:
+    if page is None or not page.has_more or not page.data:
+        return None
+    return fetch_page(client, after=page.data[-1].id)
+
+
+def list_all_stores(client: httpx.Client) -> Iterator[VectorStore]:
+    pages: Final = accumulate(
+        repeat(None), lambda page, _: fetch_next_page(client, page), initial=fetch_page(client, after=None)
+    )
+    for page in takewhile(lambda page: page is not None, pages):
+        yield from page.data
+
+
+def retrieve_stores(client: httpx.Client, store_ids: tuple[str, ...]) -> Iterator[VectorStore]:
+    for store_id in store_ids:
+        yield VectorStore.model_validate_json(client.get(f"/vector_stores/{store_id}").raise_for_status().content)
+
+
+def describe_http_error(response: httpx.Response) -> str:
+    try:
+        message: Final = OpenAIErrorResponse.model_validate_json(response.content).error.message
+    except ValidationError:
+        return f"HTTP {response.status_code}"
+    return f"HTTP {response.status_code}: {message}"
+
+
+def delete_keys(client: httpx.Client, store_id: str, keys: tuple[str, ...]) -> StoreResult:
+    try:
+        response: Final = client.post(f"/vector_stores/{store_id}", json={"metadata": dict.fromkeys(keys)})
+    except httpx.TransportError as error:
+        return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=type(error).__name__)
+    if response.is_success:
+        return StoreResult(store_id=store_id, matched_keys=keys, status="updated")
+    return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=describe_http_error(response))
+
+
+def clean_store(client: httpx.Client, store: VectorStore, options: Options) -> StoreResult | None:
+    keys: Final = matching_keys(store, options.extra_keys)
+    if not keys:
+        return None
+    if not options.apply:
+        return StoreResult(store_id=store.id, matched_keys=keys, status="dry_run")
+    return delete_keys(client, store.id, keys)
+
+
+def format_result(result: StoreResult) -> str:
+    key_list: Final = ", ".join(result.matched_keys)
+    count: Final = len(result.matched_keys)
+    match result.status:
+        case "dry_run":
+            return f"{result.store_id}: would delete {count} keys: {key_list}"
+        case "updated":
+            return f"{result.store_id}: deleted {count} keys: {key_list}"
+        case "failed":
+            return f"{result.store_id}: failed to delete {count} keys ({result.error}): {key_list}"
+
+
+def clean_and_report(
+    client: httpx.Client, stores: Iterator[VectorStore], options: Options, out: TextIO
+) -> Iterator[StoreResult | None]:
+    for store in stores:
+        result: Final = clean_store(client, store, options)
+        if result is not None:
+            print(format_result(result), file=out)
+        yield result
+
+
+def run(options: Options, client: httpx.Client, out: TextIO) -> int:
+    stores: Final = (
+        retrieve_stores(client, options.vector_store_ids) if options.vector_store_ids else list_all_stores(client)
+    )
+    scanned: Final = tuple(clean_and_report(client, stores, options, out))
+    results: Final = tuple(result for result in scanned if result is not None)
+    updated: Final = sum(1 for result in results if result.status == "updated")
+    failed: Final = sum(1 for result in results if result.status == "failed")
+    mode: Final = "applied" if options.apply else "dry run, pass --apply to delete"
+    print(
+        f"scanned={len(scanned)} matched={len(results)} updated={updated} failed={failed} ({mode})",
+        file=out,
+    )
+    return 1 if failed else 0
+
+
+def parse_options(argv: Sequence[str] | None) -> Options:
+    parser: Final = argparse.ArgumentParser(
+        description="Remove LiteLLM proxy-written keys from OpenAI vector store metadata. Dry run by default."
+    )
+    parser.add_argument("--api-key", default=os.environ.get("OPENAI_API_KEY"), help="defaults to $OPENAI_API_KEY")
+    parser.add_argument("--api-base", default=DEFAULT_API_BASE)
+    parser.add_argument("--vector-store-id", action="append", default=[], help="repeatable, defaults to every store")
+    parser.add_argument("--extra-key", action="append", default=[], help="repeatable, another key name to delete")
+    parser.add_argument("--apply", action="store_true", help="delete the matching keys instead of listing them")
+    args: Final = parser.parse_args(argv)
+    if not args.api_key:
+        parser.error("--api-key or OPENAI_API_KEY is required")
+    return Options(
+        api_key=args.api_key,
+        api_base=args.api_base.rstrip("/"),
+        vector_store_ids=tuple(args.vector_store_id),
+        extra_keys=frozenset(args.extra_key),
+        apply=args.apply,
+    )
+
+
+def main(argv: Sequence[str] | None = None, transport: httpx.BaseTransport | None = None) -> int:
+    options: Final = parse_options(argv)
+    with httpx.Client(
+        base_url=options.api_base,
+        headers={"Authorization": f"Bearer {options.api_key}"},
+        timeout=60.0,
+        transport=transport,
+    ) as client:
+        try:
+            return run(options, client, sys.stdout)
+        except httpx.HTTPStatusError as error:
+            sys.stderr.write(f"could not read vector stores: {describe_http_error(error.response)}\n")
+            return 2
+        except httpx.TransportError as error:
+            sys.stderr.write(f"could not reach {options.api_base}: {type(error).__name__}\n")
+            return 2
+        except ValidationError:
+            sys.stderr.write(f"unexpected vector store response shape from {options.api_base}\n")
+            return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -3,6 +3,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
 """
 
 import asyncio
+import json
 from collections.abc import Mapping, Sequence
 from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
@@ -227,7 +228,7 @@ class CheckBatchCost:
             "so it will no longer be polled"
         )
 
-    async def _claim_job_for_costing(self, job: "ManagedObjectRow") -> bool:
+    async def _claim_job_for_costing(self, job: "ManagedObjectRow", response: "LiteLLMBatch") -> bool:
         """
         Atomically flip batch_processed from false to true, returning whether this pod won
         the row. Every pod and uvicorn worker schedules its own poller against the shared
@@ -241,10 +242,16 @@ class CheckBatchCost:
         """
         if not self._has_batch_processed_column:
             return True
+        from litellm_enterprise.proxy.hooks.managed_files import postgres_storable_json
+
         try:
             claimed: Final = await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": False},
-                data={"batch_processed": True},
+                data={
+                    "batch_processed": True,
+                    "status": response.status if response.status != "completed" else "complete",
+                    "file_object": postgres_storable_json(response),
+                },
             )
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -253,7 +260,9 @@ class CheckBatchCost:
             return False
         return claimed > 0
 
-    async def _release_job_claim(self, job: "ManagedObjectRow") -> None:
+    async def _release_job_claim(
+        self, job: "ManagedObjectRow", original_status: str | None, original_file_object: object
+    ) -> None:
         """Give a claimed row back once billing it failed, so a later poll cycle retries it.
 
         Safe to match on batch_processed=True: while this poller is active the retrieve
@@ -265,7 +274,11 @@ class CheckBatchCost:
         try:
             await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": True},
-                data={"batch_processed": False},
+                data={
+                    "batch_processed": False,
+                    "status": original_status,
+                    "file_object": json.dumps(original_file_object),
+                },
             )
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -564,8 +577,6 @@ class CheckBatchCost:
 
     @staticmethod
     def _get_input_file_id(job: "ManagedObjectRow") -> Optional[str]:
-        import json
-
         from litellm.types.utils import LiteLLMBatch
 
         file_object: Final = job.file_object
@@ -718,8 +729,7 @@ class CheckBatchCost:
                         )
                     except Exception as _e:
                         verbose_proxy_logger.warning(
-                            f"CheckBatchCost: failed to create managed file ID for "
-                            f"{_file_attr}={_raw_file_id!r}: {_e}"
+                            f"CheckBatchCost: failed to create managed file ID for {_file_attr}={_raw_file_id!r}: {_e}"
                         )
 
         # Pass the deployment's router-registered pricing (litellm_params custom
@@ -755,8 +765,9 @@ class CheckBatchCost:
                 failed_requests=output_file_result.failed_requests + error_file_failed_requests,
             )
         )
+        logging_model: Final = batch_result.models[0] if batch_result.models else litellm_model_name or model_id
         logging_obj = LiteLLMLogging(
-            model=batch_result.models[0],
+            model=logging_model,
             messages=[{"role": "user", "content": "<retrieve_batch>"}],
             stream=False,
             call_type="aretrieve_batch",
@@ -787,7 +798,9 @@ class CheckBatchCost:
             custom_llm_provider=str(llm_provider) if llm_provider else None,
         )
 
-        if not await self._claim_job_for_costing(job):
+        original_status: Final = job.status
+        original_file_object: Final = job.file_object
+        if not await self._claim_job_for_costing(job, response.model_copy(update={"usage": batch_result.usage})):
             verbose_proxy_logger.info(
                 f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
                 "in this window, so its cost is already being tracked there"
@@ -806,7 +819,7 @@ class CheckBatchCost:
                 batch_completion_cost=batch_result.completion_cost,
             )
         except Exception:
-            await self._release_job_claim(job)
+            await self._release_job_claim(job, original_status, original_file_object)
             raise
 
         # Record batch duration (completed_at - created_at)
@@ -1099,10 +1112,7 @@ class CheckBatchCost:
                 continue
 
             ## RETRIEVE THE BATCH JOB OUTPUT FILE
-            if (
-                response.status in PROVIDER_TERMINAL_BATCH_STATUSES
-                and response.output_file_id is not None
-            ):
+            if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None:
                 try:
                     tracked = await self._track_completed_batch_cost(
                         job=job,
@@ -1133,24 +1143,19 @@ class CheckBatchCost:
                 # Track this job for the final metrics summary
                 processed_models.append(tracked)
 
-                # mark the job as complete
-                try:
-                    update_data: dict = {
-                        "status": response.status if response.status != "completed" else "complete",
-                        "file_object": response.model_dump_json(),
-                    }
-                    if self._has_batch_processed_column:
-                        update_data["batch_processed"] = True
-                    await _managed_object_table(self.prisma_client).update(
-                        where={"id": job.id},
-                        data=update_data,
-                    )
-                except Exception as db_err:
-                    verbose_proxy_logger.error(
-                        f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
-                    )
-                    if not self._has_batch_processed_column:
-                        # nothing claimed the row, so it stays in the poll page and its lines wait for that retry
+                if not self._has_batch_processed_column:
+                    try:
+                        await _managed_object_table(self.prisma_client).update(
+                            where={"id": job.id},
+                            data={
+                                "status": response.status if response.status != "completed" else "complete",
+                                "file_object": response.model_dump_json(),
+                            },
+                        )
+                    except Exception as db_err:
+                        verbose_proxy_logger.error(
+                            f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
+                        )
                         continue
                 self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)
 

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -6,6 +6,7 @@ import base64
 import json
 import time
 from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from itertools import chain
 from types import MappingProxyType
 from typing import (
     TYPE_CHECKING,
@@ -416,7 +417,7 @@ def _storable_json_value(value: JsonValue) -> JsonValue:
     return value
 
 
-def _postgres_storable_json(file_object: BaseModel) -> str:
+def postgres_storable_json(file_object: BaseModel) -> str:
     dumped: Final = file_object.model_dump_json()
     if "\\u0000" not in dumped:
         return dumped
@@ -479,6 +480,13 @@ def _with_managed_background_marker(request: Mapping[str, object]) -> dict[str,
 
 
 _MANAGED_FILES_TARGET: Final = "managed_files"
+_PROVIDER_FILE_ID_KWARGS: Final = MappingProxyType(
+    {
+        CallTypes.acreate_batch: ("input_file_id",),
+        CallTypes.acreate_fine_tuning_job: ("training_file", "validation_file"),
+    }
+)
+_PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDER_FILE_ID_KWARGS.values())))
 _PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
 
 
@@ -650,7 +658,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         )
         # FIX: Update status and file_object on every operation to keep state in sync
         update_columns: Final = {
-            "file_object": _postgres_storable_json(stored_object),
+            "file_object": postgres_storable_json(stored_object),
             "status": file_object.status,
             "updated_by": user_api_key_dict.user_id,
         }
@@ -666,7 +674,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             data={
                 "create": {
                     "unified_object_id": unified_object_id,
-                    "file_object": _postgres_storable_json(stored_object),
+                    "file_object": postgres_storable_json(stored_object),
                     "model_object_id": model_object_id,
                     "file_purpose": file_purpose,
                     "created_by": resolve_resource_owner_id(user_api_key_dict),
@@ -702,7 +710,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "unified_object_id": response_id,
                     "model_object_id": decrypted_response_id(response_id),
                     "file_purpose": "response",
-                    "file_object": _postgres_storable_json(_without_model_output(response)),
+                    "file_object": postgres_storable_json(_without_model_output(response)),
                     "status": PENDING_RESPONSE_STATUSES[0],
                     "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
@@ -1562,6 +1570,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 if isinstance(file_id, str) and file_id
             )
             await self.check_file_ids_access(fine_tuning_file_ids, user_api_key_dict)
+            if fine_tuning_file_ids:
+                data["model_file_id_mapping"] = await self.get_model_file_id_mapping(
+                    fine_tuning_file_ids, user_api_key_dict.parent_otel_span
+                )
 
         return data
 
@@ -1576,24 +1588,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         if request_kwargs is None:
             return healthy_deployments
 
-        input_file_id = cast(Optional[str], request_kwargs.get("input_file_id"))
-        model_file_id_mapping = cast(
-            Optional[Dict[str, Dict[str, str]]],
-            request_kwargs.get("model_file_id_mapping"),
+        request: Final = cast(Mapping[str, object], request_kwargs)
+        model_file_id_mapping: Final = cast(Mapping[str, Mapping[str, str]], request.get("model_file_id_mapping") or {})
+        requested_file_ids: Final = (request.get(key) for key in _PROVIDER_FILE_ID_KEYS)
+        model_ids_per_file: Final = tuple(
+            frozenset(model_file_id_mapping[file_id])
+            for file_id in requested_file_ids
+            if isinstance(file_id, str) and file_id in model_file_id_mapping
+        )
+        allowed_model_ids: Final[frozenset[str]] = (
+            model_ids_per_file[0].intersection(*model_ids_per_file[1:]) if model_ids_per_file else frozenset()
         )
-        allowed_model_ids = []
-        if input_file_id and model_file_id_mapping:
-            model_id_dict = model_file_id_mapping.get(input_file_id, {})
-            allowed_model_ids = list(model_id_dict.keys())
 
         if len(allowed_model_ids) == 0:
             return healthy_deployments
 
-        return [
+        holders: Final[list[dict[str, object]]] = [
             deployment
             for deployment in healthy_deployments
             if deployment.get("model_info", {}).get("id") in allowed_model_ids
         ]
+        return holders if holders or "input_file_id" in request else healthy_deployments
 
     async def async_pre_call_deployment_hook(
         self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
@@ -1601,25 +1616,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         """
         Allow modifying the request just before it's sent to the deployment.
         """
-        accessor_key: Optional[str] = None
-        if call_type and call_type == CallTypes.acreate_batch:
-            accessor_key = "input_file_id"
-        elif call_type and call_type == CallTypes.acreate_fine_tuning_job:
-            accessor_key = "training_file"
-        else:
+        accessor_keys: Final = _PROVIDER_FILE_ID_KWARGS.get(call_type) if call_type else None
+        if accessor_keys is None:
             return kwargs
 
-        if accessor_key:
+        model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
+        # model_info may be at top-level or nested under litellm_metadata
+        # (batch/file operations use litellm_metadata)
+        model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
+        if model_id is None:
+            model_id = cast(
+                Optional[str],
+                kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
+            )
+        for accessor_key in accessor_keys:
             input_file_id = cast(Optional[str], kwargs.get(accessor_key))
-            model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
-            # model_info may be at top-level or nested under litellm_metadata
-            # (batch/file operations use litellm_metadata)
-            model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
-            if model_id is None:
-                model_id = cast(
-                    Optional[str],
-                    kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
-                )
             mapped_file_id: Optional[str] = None
             if input_file_id and model_file_id_mapping and model_id:
                 mapped_file_id = model_file_id_mapping.get(input_file_id, {}).get(model_id, None)
@@ -1813,7 +1824,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     ),
                 )
 
-    async def get_model_file_id_mapping(self, file_ids: List[str], litellm_parent_otel_span: Span) -> dict:
+    async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict:
         """
         Get model-specific file IDs for a list of proxy file IDs.
         Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.77"
+version = "0.1.78"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.77"
+version = "0.1.78"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

helm/litellm-helm/README.md

@@ -70,11 +70,18 @@ proxyConfigMap:
 proxy_config:
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
+  model_list: []
+```
+
+The chart ships no models, so add them under `proxy_config.model_list` or from the Admin UI with `store_model_in_db: true`
+
+```
+proxy_config:
   model_list:
-    - model_name: gpt-3.5-turbo
+    - model_name: gpt-5.6
       litellm_params:
-        model: gpt-3.5-turbo
-        api_key: eXaMpLeOnLy
+        model: openai/gpt-5.6
+        api_key: os.environ/OPENAI_API_KEY
 ```
 
 #### Example using existing `proxyConfigMap` instead of creating it:

helm/litellm-helm/tests/deployment_tests.yaml

@@ -270,6 +270,32 @@ tests:
       - matchRegex:
           path: data["config.yaml"]
           pattern: '(?m)^\s*health_check_details:\s*false$'
+  - it: should not render any placeholder models when the user sets proxy_config without model_list
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.general_settings.store_model_in_db: true
+    asserts:
+      - notMatchRegex:
+          path: data["config.yaml"]
+          pattern: 'model_name:'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'store_model_in_db: true'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'master_key: os.environ/PROXY_MASTER_KEY'
+  - it: should render a user-supplied model_list as the only models
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.model_list:
+        - model_name: my-model
+          litellm_params:
+            model: openai/my-model
+            api_key: os.environ/OPENAI_API_KEY
+    asserts:
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'model_list:\n- litellm_params:\n    api_key: os.environ/OPENAI_API_KEY\n    model: openai/my-model\n  model_name: my-model\n$'
   - it: should allow overriding liveness, readiness, and startup probes
     template: deployment.yaml
     set:

helm/litellm-helm/values.yaml

@@ -180,17 +180,7 @@ proxyConfigMap:
 #  Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
 #  Reference: https://docs.litellm.ai/docs/proxy/configs
 proxy_config:
-  model_list:
-    # At least one model must exist for the proxy to start.
-    - model_name: gpt-3.5-turbo
-      litellm_params:
-        model: gpt-3.5-turbo
-        api_key: eXaMpLeOnLy
-    - model_name: fake-openai-endpoint
-      litellm_params:
-        model: openai/fake
-        api_key: fake-key
-        api_base: https://exampleopenaiendpoint-production.up.railway.app/
+  model_list: []
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
 

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.109"
+version = "0.4.110"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.109"
+version = "0.4.110"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/Cargo.lock

@@ -4083,6 +4083,7 @@ dependencies = [
 name = "litellm-inference-responses"
 version = "0.1.0"
 dependencies = [
+ "base64 0.22.1",
  "bytes",
  "futures-util",
  "litellm-auth",

litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs

@@ -94,6 +94,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
     "docker_model_runner",
     "ragflow",
     "pinstripes",
+    "openinfer",
     "darkbloom",
     "meta",
     "cognition",

litellm-rust/crates/inference-chat/src/common_utils.rs

@@ -6,9 +6,11 @@ use litellm_llms::{
     bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
     openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG,
 };
+use litellm_llms_types::formats::chat_completions::ChatMessage;
 use serde_json::{Map, Value};
 
 use super::Error;
+use crate::types::{ChatCompletionsCall, ResolvedChatCompletionsRequest};
 
 const HEADER_CONTEXT: &str = "chat completions";
 
@@ -42,3 +44,58 @@ pub(super) fn string_headers(
 ) -> Result<Vec<(String, String)>, Error> {
     shared_string_headers(HEADER_CONTEXT, extra_headers).map_err(Error::from)
 }
+
+pub(super) fn resolve_request(
+    call: ChatCompletionsCall,
+    config: &'static dyn BaseConfig,
+) -> Result<ResolvedChatCompletionsRequest, Error> {
+    let messages = serde_json::from_value::<Vec<ChatMessage>>(call.messages).map_err(|err| {
+        Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
+            "chat completions messages",
+            err,
+        ))
+    })?;
+    if messages.is_empty() {
+        return Err(Error::InvalidRequest(
+            "chat completions requires at least one message".into(),
+        ));
+    }
+    if let Some(reason) = config.unsupported_reason(&messages, &call.optional_params) {
+        return Err(Error::Unsupported(reason.0));
+    }
+    Ok(ResolvedChatCompletionsRequest {
+        messages,
+        optional_params: call.optional_params,
+        api_key: call.api_key,
+        api_base: call.api_base,
+        extra_headers: call.extra_headers,
+        timeout: call.timeout,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::chat_completions_provider;
+    use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+
+    #[rstest]
+    #[case::anthropic(LlmProviders::Anthropic, ("anthropic-version", "2023-06-01"))]
+    #[case::bedrock(LlmProviders::Bedrock, ("Content-Type", "application/json"))]
+    fn resolves_supported_provider_configs(
+        #[case] provider: LlmProviders,
+        #[case] expected_header: (&str, &str),
+    ) {
+        let config = chat_completions_provider(provider)
+            .expect("provider has a chat completions config")
+            .config();
+        assert!(config.default_headers().contains(&expected_header));
+    }
+
+    #[rstest]
+    #[case::openai(LlmProviders::Openai)]
+    fn rejects_providers_without_a_chat_config(#[case] provider: LlmProviders) {
+        assert!(chat_completions_provider(provider).is_none());
+    }
+}

litellm-rust/crates/inference-chat/src/handler.rs

@@ -3,44 +3,66 @@ use std::time::Duration;
 
 use litellm_auth::AuthServices;
 use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
-use litellm_http::{Client, outbound::OutboundRequest, request::truncate_error_body};
+use litellm_http::{
+    Client,
+    outbound::OutboundRequest,
+    request::{truncate_error_body, with_default_headers},
+};
 use litellm_llms::base_llm::{
-    auth::{Authenticated, resolve_auth},
+    auth::{Authenticated, ValidatedEnvironment, resolve_auth},
+    chat::transformation::BaseConfig,
     chat::transformation::ProviderChatResponseData,
 };
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
 use super::Error;
-use crate::{constants::CHAT_COMPLETIONS_TIMEOUT_SECS, types::ProviderChatCompletionsRequest};
+use crate::{
+    common_utils::string_headers, constants::CHAT_COMPLETIONS_TIMEOUT_SECS,
+    types::ResolvedChatCompletionsRequest,
+};
 
+#[expect(clippy::too_many_arguments)] // mirrors BaseLLMHTTPHandler.completion, which takes each stage input separately
 pub(super) async fn execute(
     http: &Client,
     auth: &AuthServices,
-    request: ProviderChatCompletionsRequest,
+    config: &'static dyn BaseConfig,
+    provider: litellm_inference::provider::ResolvedProvider<'_>,
+    call: ResolvedChatCompletionsRequest,
+    secrets: Secrets,
     cache: Option<litellm_cache_response::ScopedCache>,
     cache_options: Option<litellm_cache_response::CachePolicy>,
     interceptors: &impl Interceptors<Error>,
     observers: Option<&ObservationSender>,
 ) -> Result<ChatCompletionsResponse, Error> {
-    let ProviderChatCompletionsRequest {
-        model,
-        custom_llm_provider,
-        config,
-        url,
-        body,
-        optional_params,
-        environment,
-        secrets,
-        timeout,
-        api_key,
-    } = request;
+    let validated = config.validate_environment(
+        string_headers(call.extra_headers)?,
+        call.api_key.as_deref(),
+        provider.model,
+        &call.optional_params,
+        &|key| secrets.get(key),
+    )?;
+    let url = config.get_complete_url(
+        call.api_base.as_deref(),
+        provider.model,
+        &call.optional_params,
+        &|key| secrets.get(key),
+    )?;
+    let body = config
+        .transform_request(provider.model, call.messages, call.optional_params.clone())?
+        .body;
+    let environment = ValidatedEnvironment {
+        headers: with_default_headers(validated.headers, config.default_headers()),
+        auth: validated.auth,
+    };
+    let provider_name = <&'static str>::from(provider.provider);
     let context = RequestContext {
-        model: model.clone(),
-        custom_llm_provider,
-        optional_params: Value::Object(optional_params),
+        model: provider.model.to_owned(),
+        custom_llm_provider: provider_name.to_owned(),
+        optional_params: Value::Object(call.optional_params),
         secret_fields: Vec::new(),
-        api_key,
+        api_key: call.api_key.map(litellm_auth::SecretValue::new),
     };
     let authenticated = resolve_auth(auth, environment, &|key| secrets.get(key)).await?;
     let identity = litellm_host::interceptors::ProviderIdentity {
@@ -76,7 +98,7 @@ pub(super) async fn execute(
                 },
                 wire.url,
                 &wire.body,
-                timeout,
+                call.timeout,
             )?;
 
             let response = litellm_inference::outbound::send(outbound, http)
@@ -121,7 +143,7 @@ pub(super) async fn execute(
                 ))
             })?;
             config
-                .transform_response(&model, ProviderChatResponseData { body })
+                .transform_response(provider.model, ProviderChatResponseData { body })
                 .map_err(Error::from)
                 .map_err(as_response_error)
         },
@@ -134,7 +156,7 @@ pub(super) async fn execute(
 ///
 /// A config reports the same variants on either side of the call: a missing
 /// field or an unsupported block can mean "this request cannot be translated"
-/// during prepare and "this response cannot be normalized" here. Only the
+/// before the request is sent and "this response cannot be normalized" here. Only the
 /// second kind has already been billed, and a host that keeps a reference
 /// implementation must not retry those, so collapse them to one variant that
 /// can only mean the provider was already called.
@@ -178,9 +200,10 @@ mod tests {
 
     use super::*;
     use crate::{
-        prepare::{prepare_provider_request, resolve_request},
-        types::ChatCompletionsRequest,
+        common_utils::{chat_completions_provider, resolve_request},
+        types::{ChatCompletionsCall, ResolvedChatCompletionsRequest},
     };
+    use litellm_inference::provider::resolve_llm_provider;
 
     const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
 
@@ -217,22 +240,33 @@ mod tests {
         }
     }
 
-    fn prepared(api_base: &str) -> ProviderChatCompletionsRequest {
-        prepare_provider_request(
-            resolve_request(ChatCompletionsRequest {
-                model: "anthropic/claude-sonnet-4-5",
+    fn prepared(
+        api_base: &str,
+    ) -> (
+        litellm_inference::provider::ResolvedProvider<'static>,
+        &'static dyn BaseConfig,
+        ResolvedChatCompletionsRequest,
+    ) {
+        let provider =
+            resolve_llm_provider("anthropic/claude-sonnet-4-5", None, "chat completions").unwrap();
+        let config = chat_completions_provider(provider.provider)
+            .expect("provider has a chat completions config")
+            .config();
+        let request = resolve_request(
+            ChatCompletionsCall {
+                model: "anthropic/claude-sonnet-4-5".into(),
                 messages: json!([{"role": "user", "content": "hi"}]),
                 optional_params: json!({"max_tokens": 16}).as_object().unwrap().clone(),
-                api_key: Some("sk-test"),
-                api_base: Some(api_base),
+                api_key: Some("sk-test".into()),
+                api_base: Some(api_base.into()),
                 custom_llm_provider: None,
                 extra_headers: None,
                 timeout: None,
-            })
-            .unwrap(),
-            std::sync::Arc::new(|_: &str| None),
+            },
+            config,
         )
-        .unwrap()
+        .unwrap();
+        (provider, config, request)
     }
 
     #[rstest]
@@ -247,10 +281,14 @@ mod tests {
             .await;
         let interceptors = RecordingHooks::default();
 
+        let (provider, config, request) = prepared(&upstream.uri());
         execute(
             &Client::plain_for_test(),
             &AuthServices::default(),
-            prepared(&upstream.uri()),
+            config,
+            provider,
+            request,
+            std::sync::Arc::new(|_: &str| None),
             None,
             None,
             &interceptors,
@@ -288,10 +326,14 @@ mod tests {
             .await;
         let interceptors = RecordingHooks::default();
 
+        let (provider, config, request) = prepared(&upstream.uri());
         let error = execute(
             &Client::plain_for_test(),
             &AuthServices::default(),
-            prepared(&upstream.uri()),
+            config,
+            provider,
+            request,
+            std::sync::Arc::new(|_: &str| None),
             None,
             None,
             &interceptors,

litellm-rust/crates/inference-chat/src/lib.rs

@@ -5,14 +5,13 @@ pub use litellm_inference::RouteError as Error;
 mod common_utils;
 pub mod constants;
 pub(crate) mod handler;
-mod prepare;
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
-use prepare::{prepare_provider_request, resolve_request};
 
 use litellm_auth::AuthServices;
+use litellm_inference::provider::resolve_llm_provider;
 use litellm_secrets::source::SecretSource;
 use std::sync::Arc;
-use types::ChatCompletionsRequest;
+use types::{ChatCompletionsCall, ChatCompletionsRequest};
 
 #[derive(Clone)]
 pub struct ChatCompletionsRoute {
@@ -55,7 +54,7 @@ impl ChatCompletionsRoute {
         } = options.into();
         litellm_host::lifecycle::observe_unary(
             observers.clone(),
-            self.run_call(
+            self.run(
                 request.into(),
                 cache_options,
                 interceptors,
@@ -65,30 +64,52 @@ impl ChatCompletionsRoute {
         .await
     }
 
+    #[tracing::instrument(name = "litellm.route", skip_all, fields(
+        route = "chat_completions",
+        model = %call.model,
+        provider,
+        resolved_model,
+        stream = false,
+        outcome
+    ))]
     async fn run(
         &self,
-        request: ChatCompletionsRequest<'_>,
+        call: ChatCompletionsCall,
         cache_options: Option<litellm_cache_response::CachePolicy>,
         interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
         observers: Option<&ObservationSender>,
     ) -> Result<ChatCompletionsResponse, Error> {
-        let resolved = resolve_request(request)?;
-        let snapshot = self
-            .secrets
-            .resolve(&resolved.config.secret_names())
-            .await?;
-        let prepared = prepare_provider_request(resolved, snapshot)?;
-        litellm_inference::diagnostic::provider(&prepared.model, &prepared.custom_llm_provider);
-        let execute: futures_util::future::BoxFuture<'_, Result<ChatCompletionsResponse, Error>> =
-            Box::pin(handler::execute(
+        litellm_inference::diagnostic::unary(async {
+            let model = call.model.clone();
+            let custom_llm_provider = call.custom_llm_provider.clone();
+            let provider =
+                resolve_llm_provider(&model, custom_llm_provider.as_deref(), "chat completions")?;
+            let config = common_utils::chat_completions_provider(provider.provider)
+                .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?
+                .config();
+            let call = common_utils::resolve_request(call, config)?;
+            litellm_inference::diagnostic::provider(
+                provider.model,
+                <&'static str>::from(provider.provider),
+            );
+            let secrets = self.secrets.resolve(&config.secret_names()).await?;
+            let execute: futures_util::future::BoxFuture<
+                '_,
+                Result<ChatCompletionsResponse, Error>,
+            > = Box::pin(handler::execute(
                 &self.http,
                 &self.auth,
-                prepared,
+                config,
+                provider,
+                call,
+                secrets,
                 self.cache.clone(),
                 cache_options,
                 interceptors,
                 observers,
             ));
-        execute.await
+            execute.await
+        })
+        .await
     }
 }

litellm-rust/crates/inference-chat/src/prepare.rs

@@ -1,638 +0,0 @@
-use litellm_auth::SecretValue;
-use litellm_core_utils::settings::Lookup;
-use litellm_http::request::with_default_headers;
-use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
-use litellm_llms_types::formats::chat_completions::ChatMessage;
-use litellm_secrets::source::Secrets;
-use serde_json::Value;
-
-use super::{
-    Error,
-    common_utils::{chat_completions_provider, string_headers},
-};
-use crate::types::{
-    ChatCompletionsRequest, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest,
-};
-use litellm_inference::provider::resolve_llm_provider;
-
-pub(super) struct ResolvedProvider {
-    pub(super) model: String,
-    pub(super) custom_llm_provider: String,
-    pub(super) config: &'static dyn BaseConfig,
-}
-
-pub(super) fn resolve_provider_config<'a>(
-    model: &'a str,
-    custom_llm_provider: Option<&'a str>,
-) -> Result<ResolvedProvider, Error> {
-    let provider_info = resolve_llm_provider(model, custom_llm_provider, "chat completions")?;
-    let config = chat_completions_provider(provider_info.provider)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?
-        .config();
-    Ok(ResolvedProvider {
-        model: provider_info.model.to_string(),
-        custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
-        config,
-    })
-}
-
-pub(super) fn parse_messages(messages: Value) -> Result<Vec<ChatMessage>, Error> {
-    serde_json::from_value(messages).map_err(|err| {
-        Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
-            "chat completions messages",
-            err,
-        ))
-    })
-}
-
-pub(super) fn resolve_request(
-    request: ChatCompletionsRequest<'_>,
-) -> Result<ResolvedChatCompletionsRequest<'_>, Error> {
-    let ResolvedProvider {
-        model,
-        custom_llm_provider,
-        config,
-    } = resolve_provider_config(request.model, request.custom_llm_provider)?;
-    let messages = parse_messages(request.messages)?;
-    if messages.is_empty() {
-        return Err(Error::InvalidRequest(
-            "chat completions requires at least one message".into(),
-        ));
-    }
-    if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) {
-        return Err(Error::Unsupported(reason.0));
-    }
-    Ok(ResolvedChatCompletionsRequest {
-        model,
-        custom_llm_provider,
-        config,
-        messages,
-        optional_params: request.optional_params,
-        api_key: request.api_key,
-        api_base: request.api_base,
-        extra_headers: request.extra_headers,
-        timeout: request.timeout,
-    })
-}
-
-fn validate_environment(
-    request: &ResolvedChatCompletionsRequest<'_>,
-    model: &str,
-    config: &dyn BaseConfig,
-    secrets: &dyn Lookup,
-) -> Result<ValidatedEnvironment, Error> {
-    let env_lookup = |key: &str| secrets.get(key);
-    let forwarded = string_headers(request.extra_headers.clone())?;
-    let validated = config.validate_environment(
-        forwarded,
-        request.api_key,
-        model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    Ok(ValidatedEnvironment {
-        headers: with_default_headers(validated.headers, config.default_headers()),
-        auth: validated.auth,
-    })
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) fn prepare_provider_request(
-    request: ResolvedChatCompletionsRequest<'_>,
-    secrets: Secrets,
-) -> Result<ProviderChatCompletionsRequest, Error> {
-    let environment =
-        validate_environment(&request, &request.model, request.config, secrets.as_ref())?;
-    let model = request.model;
-    let config = request.config;
-    let env_lookup = |key: &str| secrets.get(key);
-    let url = config.get_complete_url(
-        request.api_base,
-        &model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    let transformed =
-        config.transform_request(&model, request.messages, request.optional_params.clone())?;
-
-    Ok(ProviderChatCompletionsRequest {
-        model,
-        custom_llm_provider: request.custom_llm_provider,
-        config,
-        url,
-        body: transformed.body,
-        optional_params: request.optional_params,
-        environment,
-        secrets,
-        timeout: request.timeout,
-        api_key: request.api_key.map(|key| SecretValue::new(key.to_string())),
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use litellm_auth::CredentialPlacement;
-    use litellm_llms::base_llm::auth::{AuthScheme, resolve_auth};
-    use serde_json::{Map, Value, json};
-
-    use super::{prepare_provider_request, resolve_request};
-    use crate::{
-        Error,
-        types::{ChatCompletionsRequest, ProviderChatCompletionsRequest},
-    };
-
-    fn prepare_chat_completions_call(
-        request: ChatCompletionsRequest<'_>,
-    ) -> Result<ProviderChatCompletionsRequest, Error> {
-        prepare_provider_request(
-            resolve_request(request)?,
-            std::sync::Arc::new(|_: &str| None),
-        )
-    }
-
-    /// The headers as they go on the wire, credential applied.
-    fn wire_headers(prepared: &ProviderChatCompletionsRequest) -> Vec<(String, String)> {
-        tokio::runtime::Builder::new_current_thread()
-            .build()
-            .unwrap()
-            .block_on(resolve_auth(
-                &litellm_auth::AuthServices::default(),
-                prepared.environment.clone(),
-                &|_| None,
-            ))
-            .unwrap()
-            .headers
-    }
-
-    fn request<'a>(
-        model: &'a str,
-        provider: Option<&'a str>,
-        messages: Value,
-        optional_params: Value,
-    ) -> ChatCompletionsRequest<'a> {
-        ChatCompletionsRequest {
-            model,
-            messages,
-            optional_params: match optional_params {
-                Value::Object(map) => map,
-                other => panic!("params must be an object, got {other}"),
-            },
-            api_key: Some("sk-test"),
-            api_base: None,
-            custom_llm_provider: provider,
-            extra_headers: None,
-            timeout: None,
-        }
-    }
-
-    fn preparation_error(request: ChatCompletionsRequest<'_>) -> Error {
-        prepare_chat_completions_call(request)
-            .err()
-            .expect("request preparation should fail")
-    }
-
-    #[test]
-    fn resolves_the_provider_from_the_model_prefix() {
-        let prepared = prepare_chat_completions_call(request(
-            "anthropic/claude-sonnet-4-5",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"max_tokens": 16}),
-        ))
-        .expect("prepares");
-        assert_eq!(prepared.model, "claude-sonnet-4-5");
-        assert_eq!(prepared.url, "https://api.anthropic.com/v1/messages");
-        assert_eq!(prepared.body["model"], json!("claude-sonnet-4-5"));
-    }
-
-    #[test]
-    fn strips_an_explicit_provider_prefix_from_the_model() {
-        let prepared = prepare_chat_completions_call(request(
-            "anthropic/claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        ))
-        .expect("prepares");
-        assert_eq!(prepared.model, "claude-sonnet-4-5");
-    }
-
-    #[test]
-    fn adds_the_auth_and_default_headers() {
-        let prepared = prepare_chat_completions_call(request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        ))
-        .expect("prepares");
-        assert!(
-            wire_headers(&prepared).contains(&("x-api-key".to_string(), "sk-test".to_string()))
-        );
-        assert!(
-            wire_headers(&prepared)
-                .contains(&("anthropic-version".to_string(), "2023-06-01".to_string()))
-        );
-        assert!(matches!(
-            prepared.environment.auth,
-            AuthScheme::Credential {
-                placement: CredentialPlacement::Header("x-api-key"),
-                ..
-            }
-        ));
-    }
-
-    #[test]
-    fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
-        // Python builds `{**headers, **anthropic_headers}`, so the deployment's key
-        // overwrites a forwarded one. Honouring the caller's would let whoever sends
-        // the request choose the Anthropic principal it bills to.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "X-Api-Key".to_string(),
-            json!("sk-caller"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .collect();
-        assert_eq!(keys.len(), 1, "got {:?}", headers);
-        assert_eq!(keys[0].1, "sk-test");
-    }
-
-    #[test]
-    fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
-        // Anthropic's `validate_environment` pops `x-api-key` and sets `authorization`
-        // for an OAuth token, so re-adding the key here would put the credential into
-        // a header the host removed on purpose.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([
-            (
-                "Authorization".to_string(),
-                json!("Bearer sk-ant-oat01-token"),
-            ),
-            ("X-Api-Key".to_string(), json!("sk-caller")),
-        ]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        assert!(
-            !wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("x-api-key") && value == "sk-test"),
-            "the resolved key must not be applied over an OAuth bearer, got {:?}",
-            wire_headers(&prepared)
-        );
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-ant-oat01-token")
-        );
-    }
-
-    #[test]
-    fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
-        // Only an OAuth bearer replaces the credential. Python sends the deployment's
-        // `x-api-key` alongside any other forwarded `authorization`, so deferring on
-        // the mere presence of that header would drop the deployment's auth.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([
-            ("Authorization".to_string(), json!("Bearer unrelated")),
-            ("X-Api-Key".to_string(), json!("sk-caller")),
-        ]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .collect();
-        assert_eq!(keys.len(), 1, "got {:?}", headers);
-        assert_eq!(keys[0].1, "sk-test");
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer unrelated"),
-            "the unrelated authorization must survive, got {:?}",
-            wire_headers(&prepared)
-        );
-    }
-
-    #[rstest::rstest]
-    fn rejects_empty_messages_before_resolving_credentials() {
-        let call = ChatCompletionsRequest {
-            api_key: None,
-            ..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
-        };
-        assert!(matches!(preparation_error(call), Error::InvalidRequest(_)));
-    }
-
-    #[test]
-    fn rejects_an_unknown_provider() {
-        assert_eq!(
-            preparation_error(request(
-                "openai/gpt-4o",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({}),
-            )),
-            Error::InvalidProvider("openai".to_string())
-        );
-    }
-
-    #[test]
-    fn rejects_a_model_with_no_resolvable_provider() {
-        assert!(matches!(
-            preparation_error(request(
-                "claude-sonnet-4-5",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({}),
-            )),
-            Error::InvalidProvider(_)
-        ));
-    }
-
-    #[test]
-    fn rejects_an_empty_or_malformed_message_list() {
-        assert_eq!(
-            preparation_error(request(
-                "anthropic/claude-sonnet-4-5",
-                None,
-                json!([]),
-                json!({}),
-            )),
-            Error::InvalidRequest(
-                "chat completions requires at least one message"
-                    .to_string()
-                    .into()
-            )
-        );
-        assert!(matches!(
-            preparation_error(request(
-                "anthropic/claude-sonnet-4-5",
-                None,
-                json!("not a list"),
-                json!({}),
-            )),
-            Error::InvalidRequest(_)
-        ));
-    }
-
-    #[test]
-    fn rejects_non_string_extra_headers() {
-        let mut call = request(
-            "anthropic/claude-sonnet-4-5",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))]));
-        assert_eq!(
-            preparation_error(call),
-            Error::Headers(litellm_http::request::HeaderError {
-                context: "chat completions",
-                name: "x-trace".to_string(),
-                actual: "number",
-            })
-        );
-    }
-
-    #[test]
-    fn prepares_a_bedrock_call_without_resolving_credentials() {
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        );
-        call.api_key = None;
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        assert_eq!(
-            prepared.url,
-            "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
-        );
-        assert!(matches!(
-            &prepared.environment.auth,
-            AuthScheme::AwsSigV4 { region, service: "bedrock", .. } if region == "us-east-1"
-        ));
-        // SigV4 signs the serialized body, so prepare must not have added an
-        // Authorization header; the signer does it over the bytes sent.
-        assert!(
-            !prepared
-                .environment
-                .headers
-                .iter()
-                .any(|(name, _)| name.eq_ignore_ascii_case("authorization"))
-        );
-        assert_eq!(prepared.body["inferenceConfig"], json!({"maxTokens": 16}));
-    }
-
-    #[tokio::test]
-    async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
-        // Python signs only the AWS header set and reattaches the rest, so a header
-        // the caller forwarded rides along without joining the canonical request.
-        // Signing it makes Converse 403 on a deployment that works on Python.
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({
-                "maxTokens": 16,
-                "aws_access_key_id": "AKIDEXAMPLE",
-                "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
-            }),
-        );
-        // A key would resolve to a bearer token and never reach the signer.
-        call.api_key = None;
-        call.extra_headers = Some(Map::from_iter([(
-            "x-request-id".to_string(),
-            json!("abc-123"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let authenticated = resolve_auth(
-            &litellm_auth::AuthServices::default(),
-            prepared.environment,
-            &|_| None,
-        )
-        .await
-        .expect("resolves");
-        let signed = crate::handler::outbound_request(
-            authenticated,
-            prepared.url,
-            &prepared.body,
-            prepared.timeout,
-        )
-        .expect("signs");
-
-        let authorization = signed
-            .header("authorization")
-            .expect("carries an authorization header")
-            .to_string();
-        assert!(
-            authorization.starts_with("AWS4-HMAC-SHA256"),
-            "expected a SigV4 signature, got {authorization}"
-        );
-        assert!(
-            !authorization.contains("x-request-id"),
-            "forwarded header reached SignedHeaders: {authorization}"
-        );
-        // It still goes on the wire, it is just not part of the signature.
-        assert!(
-            signed
-                .headers()
-                .iter()
-                .any(|(name, value)| name == "x-request-id" && value == "abc-123"),
-            "forwarded header was dropped instead of reattached"
-        );
-    }
-
-    #[rstest::rstest]
-    #[case::authorization("Authorization")]
-    #[case::amz_date("x-amz-date")]
-    #[case::security_token("x-amz-security-token")]
-    #[case::date("Date")]
-    #[tokio::test]
-    async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
-        let call = ChatCompletionsRequest {
-            api_key: None,
-            extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
-            ..request(
-                "bedrock/us-east-1/anthropic.claude-v2",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({
-                    "maxTokens": 16,
-                    "aws_access_key_id": "AKIDEXAMPLE",
-                    "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
-                }),
-            )
-        };
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let authenticated = resolve_auth(
-            &litellm_auth::AuthServices::default(),
-            prepared.environment,
-            &|_| None,
-        )
-        .await
-        .expect("resolves");
-        let error = crate::handler::outbound_request(
-            authenticated,
-            prepared.url,
-            &prepared.body,
-            prepared.timeout,
-        )
-        .expect_err("conflicting signing headers must fail");
-        assert!(
-            matches!(error, Error::Unsupported(_)),
-            "{forwarded} returned {error:?}"
-        );
-    }
-
-    #[test]
-    fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
-        // `get_request_headers` assigns `headers["Authorization"]` unconditionally
-        // once a bearer token resolves, so the deployment's identity wins on
-        // Python. Keeping the caller's would authorize and bill the call as a
-        // different principal, and only when the deployment carries `rust: true`.
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "Authorization".to_string(),
-            json!("Bearer caller-supplied"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let authorizations: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("authorization"))
-            .map(|(_, value)| value.as_str())
-            .collect();
-        assert_eq!(
-            authorizations,
-            vec!["Bearer sk-test"],
-            "the deployment token must be the only authorization on the wire"
-        );
-    }
-
-    #[test]
-    fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
-        // The opposite precedence, and deliberate: Anthropic's own transform
-        // honours a forwarded OAuth bearer, so the Bedrock fix above must not be
-        // generalized into a rule that the configured key always wins.
-        //
-        // An OAuth bearer is the whole of that exception. This forwarded a plain
-        // `x-api-key` until round 17, which read as the same claim and was not:
-        // Python overwrites a forwarded `x-api-key` with the deployment's.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "authorization".to_string(),
-            json!("Bearer sk-ant-oat01-forwarded"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .map(|(_, value)| value.as_str())
-            .collect();
-        assert!(keys.is_empty(), "got {:?}", headers);
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-ant-oat01-forwarded")
-        );
-    }
-
-    #[test]
-    fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
-        // The configured bearer identity has its own account and quota boundary,
-        // so a request carrying one must not be signed as whatever principal the
-        // host's AWS credentials resolve to.
-        let prepared = prepare_chat_completions_call(request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        ))
-        .expect("prepares");
-        assert!(matches!(
-            &prepared.environment.auth,
-            AuthScheme::Credential { placement: CredentialPlacement::Bearer, secret }
-                if secret.expose() == "sk-test"
-        ));
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-test"),
-            "prepare did not carry the bearer token"
-        );
-    }
-}

litellm-rust/crates/inference-chat/src/route.rs

@@ -1,4 +1,3 @@
-use litellm_host::observation::ObservationSender;
 use std::convert::Infallible;
 
 use litellm_host::{
@@ -7,10 +6,7 @@ use litellm_host::{
 };
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
 
-use super::{
-    ChatCompletionsRoute, Error,
-    types::{ChatCompletionsCall, ChatCompletionsRequest},
-};
+use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall};
 
 pub struct ChatCompletions;
 
@@ -37,44 +33,12 @@ impl ChatCompletionsRoute {
             call,
             observers,
             move |call, _, interceptors, observers| async move {
-                self.run_call(call, cache_options, &interceptors, observers.as_ref())
+                self.run(call, cache_options, &interceptors, observers.as_ref())
                     .await
                     .map(CallOutput::Complete)
             },
         )
     }
-
-    #[tracing::instrument(name = "litellm.route", skip_all, fields(
-        route = "chat_completions",
-        model = %call.model,
-        provider,
-        resolved_model,
-        stream = false,
-        outcome
-    ))]
-    pub(super) async fn run_call(
-        &self,
-        call: ChatCompletionsCall,
-        cache_options: Option<litellm_cache_response::CachePolicy>,
-        interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<ChatCompletionsResponse, Error> {
-        litellm_inference::diagnostic::unary(async {
-            let request = ChatCompletionsRequest {
-                model: &call.model,
-                messages: call.messages,
-                optional_params: call.optional_params,
-                api_key: call.api_key.as_deref(),
-                api_base: call.api_base.as_deref(),
-                custom_llm_provider: call.custom_llm_provider.as_deref(),
-                extra_headers: call.extra_headers,
-                timeout: call.timeout,
-            };
-            self.run(request, cache_options, interceptors, observers)
-                .await
-        })
-        .await
-    }
 }
 
 impl litellm_inference::caching::Cachable for ChatCompletions {

litellm-rust/crates/inference-chat/src/types.rs

@@ -1,8 +1,5 @@
-use litellm_secrets::source::Secrets;
 use std::time::Duration;
 
-use litellm_auth::SecretValue;
-use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
 use litellm_llms_types::formats::chat_completions::ChatMessage;
 use serde_json::{Map, Value};
 
@@ -49,31 +46,11 @@ impl From<ChatCompletionsRequest<'_>> for ChatCompletionsCall {
     }
 }
 
-pub struct ResolvedChatCompletionsRequest<'a> {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseConfig,
+pub struct ResolvedChatCompletionsRequest {
     pub messages: Vec<ChatMessage>,
     pub optional_params: Map<String, Value>,
-    pub api_key: Option<&'a str>,
-    pub api_base: Option<&'a str>,
+    pub api_key: Option<String>,
+    pub api_base: Option<String>,
     pub extra_headers: Option<Map<String, Value>>,
     pub timeout: Option<Duration>,
 }
-
-pub struct ProviderChatCompletionsRequest {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseConfig,
-    pub url: String,
-    pub body: Value,
-    /// The route's parameters before the provider transformation, reported to the host
-    /// beside the wire request.
-    pub optional_params: Map<String, Value>,
-    /// The forwarded and default headers plus how the call authenticates; the credential
-    /// itself is applied when the request is sent.
-    pub environment: ValidatedEnvironment,
-    pub secrets: Secrets,
-    pub timeout: Option<Duration>,
-    pub api_key: Option<SecretValue>,
-}

litellm-rust/crates/inference-chat/tests/request.rs

@@ -0,0 +1,439 @@
+use std::time::Duration;
+
+use litellm_inference_chat::{Error, types::ChatCompletionsRequest};
+use rstest::rstest;
+use serde_json::{Map, Value, json};
+use wiremock::ResponseTemplate;
+
+mod support;
+use support::*;
+
+const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
+
+fn request<'a>(
+    model: &'a str,
+    provider: Option<&'a str>,
+    messages: Value,
+    optional_params: Value,
+) -> ChatCompletionsRequest<'a> {
+    let Value::Object(optional_params) = optional_params else {
+        panic!("params must be an object");
+    };
+    ChatCompletionsRequest {
+        model,
+        messages,
+        optional_params,
+        api_key: Some("sk-test"),
+        api_base: None,
+        custom_llm_provider: provider,
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(10)),
+    }
+}
+
+async fn complete(
+    request: ChatCompletionsRequest<'_>,
+) -> Result<litellm_llms_types::formats::chat_completions::ChatCompletionsResponse, Error> {
+    chat_completions_route().execute(request, &(), None).await
+}
+
+fn object(value: Value) -> Map<String, Value> {
+    let Value::Object(map) = value else {
+        panic!("expected an object");
+    };
+    map
+}
+
+fn anthropic_response() -> ResponseTemplate {
+    ResponseTemplate::new(200).set_body_raw(ANTHROPIC_MESSAGE, "application/json")
+}
+
+fn bedrock_response() -> ResponseTemplate {
+    json_response(json!({
+        "output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}},
+        "stopReason": "end_turn",
+        "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}
+    }))
+}
+
+#[rstest]
+#[tokio::test]
+async fn resolves_the_provider_from_the_model_prefix() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "anthropic/claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"max_tokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.url.path(), "/v1/messages");
+    assert_eq!(sent.json()["model"], "claude-sonnet-4-5");
+}
+
+#[rstest]
+#[tokio::test]
+async fn strips_an_explicit_provider_prefix_from_the_model() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "anthropic/claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.json()["model"],
+        "claude-sonnet-4-5"
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn adds_the_auth_and_default_headers() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
+    assert_eq!(sent.header("anthropic-version"), Some("2023-06-01"));
+}
+
+#[rstest]
+#[tokio::test]
+async fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({"X-Api-Key": "sk-caller"}))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("x-api-key"),
+        ["sk-test"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({
+            "Authorization": "Bearer sk-ant-oat01-token",
+            "X-Api-Key": "sk-caller"
+        }))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-caller"]);
+    assert_eq!(
+        sent.header_values("authorization"),
+        ["Bearer sk-ant-oat01-token"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({
+            "Authorization": "Bearer unrelated",
+            "X-Api-Key": "sk-caller"
+        }))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
+    assert_eq!(sent.header_values("authorization"), ["Bearer unrelated"]);
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_empty_messages_before_resolving_credentials() {
+    let error = complete(ChatCompletionsRequest {
+        api_key: None,
+        ..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
+    })
+    .await
+    .expect_err("empty messages are rejected");
+    assert!(matches!(error, Error::InvalidRequest(_)));
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_an_unknown_provider() {
+    assert_eq!(
+        complete(request(
+            "openai/gpt-4o",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        ))
+        .await
+        .unwrap_err(),
+        Error::InvalidProvider("openai".into())
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_a_model_with_no_resolvable_provider() {
+    assert!(matches!(
+        complete(request(
+            "claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        ))
+        .await,
+        Err(Error::InvalidProvider(_))
+    ));
+}
+
+#[rstest]
+#[case::empty(json!([]), Some("chat completions requires at least one message"))]
+#[case::malformed(json!("not a list"), None)]
+#[tokio::test]
+async fn rejects_an_empty_or_malformed_message_list(
+    #[case] messages: Value,
+    #[case] expected: Option<&str>,
+) {
+    let error = complete(request(
+        "anthropic/claude-sonnet-4-5",
+        None,
+        messages,
+        json!({}),
+    ))
+    .await
+    .unwrap_err();
+    if let Some(expected) = expected {
+        assert_eq!(error, Error::InvalidRequest(expected.to_string().into()));
+    } else {
+        assert!(matches!(error, Error::InvalidRequest(_)));
+    }
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_non_string_extra_headers() {
+    assert_eq!(
+        complete(ChatCompletionsRequest {
+            extra_headers: Some(Map::from_iter([("x-trace".to_string(), json!(7))])),
+            ..request(
+                "anthropic/claude-sonnet-4-5",
+                None,
+                json!([{"role": "user", "content": "hi"}]),
+                json!({}),
+            )
+        })
+        .await
+        .unwrap_err(),
+        Error::Headers(litellm_http::request::HeaderError {
+            context: "chat completions",
+            name: "x-trace".to_string(),
+            actual: "number",
+        })
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    let mut call = request(
+        "bedrock/us-east-1/anthropic.claude-v2",
+        None,
+        json!([{"role": "user", "content": "hi"}]),
+        json!({
+            "maxTokens": 16,
+            "aws_access_key_id": "AKIDEXAMPLE",
+            "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
+        }),
+    );
+    call.api_key = None;
+    call.api_base = Some(&base);
+    call.extra_headers = Some(Map::from_iter([(
+        "x-request-id".to_string(),
+        json!("abc-123"),
+    )]));
+    complete(call).await.expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    let authorization = sent.header("authorization").expect("signed request");
+    assert!(authorization.starts_with("AWS4-HMAC-SHA256"));
+    assert!(!authorization.contains("x-request-id"));
+    assert_eq!(sent.header("x-request-id"), Some("abc-123"));
+    assert_eq!(sent.url.path(), "/model/anthropic.claude-v2/converse");
+    let body: Value = serde_json::from_slice(&sent.body).unwrap();
+    assert_eq!(body["inferenceConfig"], json!({"maxTokens": 16}));
+    assert_eq!(
+        body["messages"],
+        json!([{"role": "user", "content": [{"text": "hi"}]}])
+    );
+}
+
+#[rstest]
+#[case::authorization("Authorization")]
+#[case::amz_date("x-amz-date")]
+#[case::security_token("x-amz-security-token")]
+#[case::date("Date")]
+#[tokio::test]
+async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
+    let upstream = wiremock::MockServer::start().await;
+    let base = upstream.uri();
+    let call = ChatCompletionsRequest {
+        api_key: None,
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({
+                "maxTokens": 16,
+                "aws_access_key_id": "AKIDEXAMPLE",
+                "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
+            }),
+        )
+    };
+    let error = complete(call)
+        .await
+        .expect_err("conflicting signing headers fail");
+    assert!(matches!(error, Error::Unsupported(_)), "{error:?}");
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(
+            "Authorization".to_string(),
+            json!("Bearer caller-supplied"),
+        )])),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"maxTokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("authorization"),
+        ["Bearer sk-test"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(
+            "authorization".to_string(),
+            json!("Bearer sk-ant-oat01-forwarded"),
+        )])),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert!(sent.header_values("x-api-key").is_empty());
+    assert_eq!(
+        sent.header_values("authorization"),
+        ["Bearer sk-ant-oat01-forwarded"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"maxTokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("authorization"),
+        ["Bearer sk-test"]
+    );
+}

litellm-rust/crates/inference-messages/AGENTS.md

@@ -7,8 +7,8 @@
   - delegate authentication policy, beta selection, payload rewriting, and response interpretation to those adapters
   - keep provider policy out of request preparation and transport handlers
   - calling a concrete provider helper for every provider is still a policy dependency
-- Route types such as `MessagesCall`, prepared requests, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
+- Route types such as `MessagesCall`, provider calls, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
 - The route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks
-- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `src/lib.rs` explicitly sequences cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
+- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `handler::execute` explicitly sequences the provider call build in Python's handler order, cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
 - Preserve the order of validation, normalization, caller-requested parameter removal, and provider transformation when that order affects observable behavior
 - Test provider dispatch, auth precedence, header handling, transformations, and responses through behavior, not source structure

litellm-rust/crates/inference-messages/src/common_utils.rs

@@ -82,17 +82,17 @@ mod tests {
     use litellm_core_utils::get_llm_provider_logic::LlmProviders;
 
     #[rstest]
-    #[case::tencent("tencent")]
-    #[case::anthropic("anthropic")]
-    #[case::azure_ai("azure_ai")]
-    #[case::bedrock("bedrock")]
-    #[case::deepseek("deepseek")]
-    #[case::edenai("edenai")]
-    #[case::minimax("minimax")]
-    #[case::vertex_ai("vertex_ai")]
-    fn provider_keeps_its_python_name(#[case] name: &str) {
-        let provider =
-            messages_provider(name.parse::<LlmProviders>().unwrap(), "claude-sonnet-4-5").unwrap();
+    #[case::tencent(LlmProviders::Tencent, "tencent")]
+    #[case::anthropic(LlmProviders::Anthropic, "anthropic")]
+    #[case::azure_ai(LlmProviders::AzureAi, "azure_ai")]
+    #[case::bedrock(LlmProviders::Bedrock, "bedrock")]
+    #[case::deepseek(LlmProviders::Deepseek, "deepseek")]
+    #[case::edenai(LlmProviders::Edenai, "edenai")]
+    #[case::minimax(LlmProviders::Minimax, "minimax")]
+    #[case::vertex_ai(LlmProviders::VertexAi, "vertex_ai")]
+    #[case::openrouter(LlmProviders::Openrouter, "openrouter")]
+    fn provider_keeps_its_python_name(#[case] provider: LlmProviders, #[case] name: &str) {
+        let provider = messages_provider(provider, "claude-sonnet-4-5").unwrap();
         assert_eq!(provider.as_str(), name);
     }
 

litellm-rust/crates/inference-messages/src/handler.rs

@@ -1,148 +1,271 @@
 use std::time::Duration;
 
 use bytes::Bytes;
-use futures_util::{StreamExt, TryStreamExt, stream::BoxStream};
-use litellm_host::interceptors::{Interceptors, ProviderIdentity, RequestContext, WireRequest};
-use litellm_http::transport::Error as TransportError;
+use futures_util::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream};
+use litellm_auth::{AuthServices, SecretValue};
+use litellm_core_utils::{
+    dot_notation_indexing::delete_nested_value,
+    get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
+};
+use litellm_host::interceptors::{
+    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
+};
+use litellm_http::{request::with_default_headers, transport::Error as TransportError};
 use litellm_llms::base_llm::{
-    auth::{Authenticated, resolve_auth},
+    auth::{Authenticated, ValidatedEnvironment, resolve_auth},
     messages::{
+        context::MessagesTransformContext,
         streaming::{self, ByteStream, StreamDecoder},
         transformation::BaseMessagesConfig,
     },
 };
-use litellm_llms_types::formats::messages::MessagesResponse;
+use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_secrets::source::Secrets;
 use litellm_tracing::ByteChunk;
 use serde_json::Value;
 
 use super::{
-    Error, MessagesCallResponse, MessagesRoute, common_utils::truncate_error_body,
-    prepare::ProviderMessagesRequest,
+    Error, MessagesCall, MessagesCallResponse, MessagesRoute,
+    common_utils::{MessagesProvider, string_headers, truncate_error_body},
+    constants::MESSAGES_TIMEOUT_SECS,
+    route::Messages,
+    types::invalid_request,
 };
-use crate::constants::MESSAGES_TIMEOUT_SECS;
-use litellm_inference::{context::CallContext, outbound::outbound_request};
+use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request};
 
-pub(super) struct ProviderCall {
-    pub identity: ProviderIdentity,
-    pub wire: WireRequest,
-    pub(super) provider: super::common_utils::MessagesProvider,
+struct ProviderCall {
+    identity: ProviderIdentity,
+    wire: WireRequest,
+    provider: MessagesProvider,
     signer: Option<litellm_auth_aws::SigV4Signer>,
     timeout: Option<Duration>,
     stream: bool,
 }
 
 impl ProviderCall {
-    pub fn cacheable(&self) -> bool {
+    fn cacheable(&self) -> bool {
         self.signer.is_none()
     }
 }
 
-impl MessagesRoute {
-    pub(super) async fn prepare_outbound(
-        &self,
-        request: ProviderMessagesRequest,
-        context: &CallContext<'_, impl Interceptors<Error>>,
-    ) -> Result<ProviderCall, Error> {
-        let ProviderMessagesRequest {
-            provider,
-            url,
-            body,
-            environment,
-            timeout,
-            api_key,
-        } = request;
-        let request_context = RequestContext {
-            model: body.model.clone(),
-            custom_llm_provider: provider.as_str().to_string(),
-            optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
-            secret_fields: Vec::new(),
-            api_key,
-        };
-        let authenticated =
-            resolve_auth(&self.auth, environment, &|key| std::env::var(key).ok()).await?;
-        let identity = ProviderIdentity {
-            model: request_context.model.clone(),
-            provider: request_context.custom_llm_provider.clone(),
-        };
-        let wire = context
-            .interceptors
-            .before_provider_request(
-                WireRequest {
-                    url,
-                    headers: authenticated.headers,
-                    body: provider
-                        .config()
-                        .wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
-                },
-                request_context,
-            )
-            .await?;
-        let stream = match wire.body.get("stream") {
-            None | Some(Value::Null) => false,
-            Some(Value::Bool(stream)) => *stream,
-            Some(value) => {
-                return Err(Error::InvalidRequest(
-                    litellm_llms::ErrorDetail::InvalidValue {
-                        field: "stream",
-                        expected: "a boolean",
-                        actual: value.clone(),
-                    },
-                ));
-            }
-        };
-        Ok(ProviderCall {
-            identity,
-            wire,
-            provider,
-            signer: authenticated.signer,
-            timeout,
-            stream,
+pub(super) async fn execute(
+    route: &MessagesRoute,
+    provider: MessagesProvider,
+    model: String,
+    call: MessagesCall,
+    secrets: Secrets,
+    context: &CallContext<'_, impl Interceptors<Error>>,
+) -> Result<MessagesCallResponse, Error> {
+    let request = provider_call(
+        &route.auth,
+        provider,
+        model,
+        call,
+        secrets.as_ref(),
+        context,
+    )
+    .boxed()
+    .await?;
+    let cache = CallCache::<Messages>::from_wire(
+        route.cache.as_ref().filter(|_| request.cacheable()),
+        context.cache,
+        &request.identity,
+        &request.wire,
+    );
+    let identity = request.identity.clone();
+    let (output, source) = match cache.lookup().await {
+        Some(hit) => hit,
+        None => (
+            call_provider(&route.http, request, context).await?,
+            ResultSource::Provider,
+        ),
+    };
+    let reported = match &output {
+        MessagesCallResponse::Complete(response) => provider.config().reported_cost(response),
+        MessagesCallResponse::Stream { .. } => None,
+    };
+    context
+        .result_ready(ExecutionFacts {
+            provider: identity,
+            cost: Cost::settle(&source, reported),
+            source: source.clone(),
         })
-    }
+        .await?;
+    Ok(cache.finish(output, &source).await)
+}
 
-    pub(super) async fn call_provider(
-        &self,
-        request: ProviderCall,
-        context: &CallContext<'_, impl Interceptors<Error>>,
-    ) -> Result<MessagesCallResponse, Error> {
-        let ProviderCall {
-            identity,
-            wire,
-            provider,
-            signer,
-            timeout,
-            stream,
-        } = request;
-        let provider_name = provider.as_str();
-        log_request_body(provider_name, stream, &wire.body);
-        let response = send(
-            &self.http,
-            Authenticated {
-                headers: wire.headers,
-                signer,
+async fn provider_call(
+    auth: &AuthServices,
+    provider: MessagesProvider,
+    model: String,
+    call: MessagesCall,
+    secrets: &(dyn Lookup + Send + Sync),
+    context: &CallContext<'_, impl Interceptors<Error>>,
+) -> Result<ProviderCall, Error> {
+    let MessagesCall {
+        body,
+        api_key,
+        api_base,
+        litellm_params,
+        extra_headers,
+        provider_specific_header,
+        timeout,
+        shaping,
+        ..
+    } = call;
+    let connection_params = litellm_router_types::LitellmParams {
+        api_base: api_base.clone().or(litellm_params.api_base.clone()),
+        ..litellm_params
+    };
+    let config = provider.config();
+    let env_lookup = |key: &str| secrets.get(key);
+
+    let sanitized = config.shape_request(
+        MessagesRequest { model, ..body },
+        shaping.settings.reasoning_auto_summary,
+    )?;
+    let trimmed =
+        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
+
+    let scoped =
+        get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
+    let forwarded = string_headers(Some(
+        extra_headers.into_iter().flatten().chain(scoped).collect(),
+    ))?;
+    let validated = config.validate_environment(
+        forwarded,
+        api_key.as_deref(),
+        &trimmed.model,
+        &trimmed.params,
+        &connection_params,
+        &env_lookup,
+    )?;
+
+    let body = config.transform_anthropic_messages_request(
+        trimmed,
+        &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
+    )?;
+    let environment = ValidatedEnvironment {
+        headers: config.request_headers(
+            with_default_headers(validated.headers, config.default_headers()),
+            &body,
+        ),
+        auth: validated.auth,
+    };
+
+    let url = config.get_complete_url(
+        api_base.as_deref(),
+        &body.model,
+        &connection_params,
+        body.params.stream == Some(true),
+        &env_lookup,
+    )?;
+
+    let request_context = RequestContext {
+        model: body.model.clone(),
+        custom_llm_provider: provider.as_str().to_string(),
+        optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
+        secret_fields: Vec::new(),
+        api_key: api_key.map(SecretValue::new),
+    };
+    let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?;
+    let identity = ProviderIdentity {
+        model: request_context.model.clone(),
+        provider: request_context.custom_llm_provider.clone(),
+    };
+    let wire = context
+        .interceptors
+        .before_provider_request(
+            WireRequest {
+                url,
+                headers: authenticated.headers,
+                body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
             },
-            &wire.url,
-            &wire.body,
-            timeout,
+            request_context,
         )
         .await?;
-        if !response.status().is_success() {
-            return Err(provider_error(response).await);
-        }
-        let config = provider.config();
-        if stream {
-            return Ok(streaming_response(
-                response,
-                config.stream_decoder(),
-                provider_name,
+    let stream = match wire.body.get("stream") {
+        None | Some(Value::Null) => false,
+        Some(Value::Bool(stream)) => *stream,
+        Some(value) => {
+            return Err(Error::InvalidRequest(
+                litellm_llms::ErrorDetail::InvalidValue {
+                    field: "stream",
+                    expected: "a boolean",
+                    actual: value.clone(),
+                },
             ));
         }
-        let text = response.text().await.map_err(network)?;
-        log_response_body(&text);
-        context.response_received(&text).await?;
-        decode_response(config, &identity.model, &text)
-            .map(|message| MessagesCallResponse::Complete(Box::new(message)))
+    };
+    Ok(ProviderCall {
+        identity,
+        wire,
+        provider,
+        signer: authenticated.signer,
+        timeout: timeout.or_else(|| shaping.settings.request_timeout()),
+        stream,
+    })
+}
+
+fn without_additional_drop_params(
+    request: MessagesRequest,
+    paths: &[String],
+) -> Result<MessagesRequest, Error> {
+    if paths.is_empty() {
+        return Ok(request);
     }
+    let params = serde_json::to_value(request.params).map_err(invalid_request)?;
+    let trimmed = paths
+        .iter()
+        .fold(params, |params, path| delete_nested_value(params, path));
+    Ok(MessagesRequest {
+        params: serde_json::from_value(trimmed).map_err(invalid_request)?,
+        ..request
+    })
+}
+
+async fn call_provider(
+    http: &litellm_http::Client,
+    request: ProviderCall,
+    context: &CallContext<'_, impl Interceptors<Error>>,
+) -> Result<MessagesCallResponse, Error> {
+    let ProviderCall {
+        identity,
+        wire,
+        provider,
+        signer,
+        timeout,
+        stream,
+    } = request;
+    let provider_name = provider.as_str();
+    log_request_body(provider_name, stream, &wire.body);
+    let response = send(
+        http,
+        Authenticated {
+            headers: wire.headers,
+            signer,
+        },
+        &wire.url,
+        &wire.body,
+        timeout,
+    )
+    .await?;
+    if !response.status().is_success() {
+        return Err(provider_error(response).await);
+    }
+    let config = provider.config();
+    if stream {
+        return Ok(streaming_response(
+            response,
+            config.stream_decoder(),
+            provider_name,
+        ));
+    }
+    let text = response.text().await.map_err(network)?;
+    log_response_body(&text);
+    context.response_received(&text).await?;
+    decode_response(config, &identity.model, &text)
+        .map(|message| MessagesCallResponse::Complete(Box::new(message)))
 }
 
 fn serialize_failure(err: serde_json::Error) -> Error {
@@ -271,10 +394,482 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) {
 
 #[cfg(test)]
 mod tests {
-    use rstest::rstest;
+    use litellm_llms_types::headers::ProviderSpecificHeaders;
+    use rstest::{fixture, rstest};
+    use serde_json::{Map, Value, json};
     use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
 
     use super::*;
+    use crate::{MessagesSettings, MessagesShaping, common_utils::messages_provider};
+    use litellm_inference::provider::resolve_llm_provider;
+
+    #[fixture]
+    fn shaping() -> MessagesShaping {
+        MessagesShaping::default()
+    }
+
+    fn body(value: Value) -> MessagesRequest {
+        serde_json::from_value(value).unwrap()
+    }
+
+    async fn provider_call_for(call: MessagesCall) -> Result<ProviderCall, Error> {
+        provider_call_with_secrets(call, &|_: &str| None).await
+    }
+
+    async fn provider_call_with_secrets(
+        call: MessagesCall,
+        secrets: &(dyn Lookup + Send + Sync),
+    ) -> Result<ProviderCall, Error> {
+        let provider = resolve_llm_provider(
+            &call.body.model,
+            call.custom_llm_provider.as_deref(),
+            "messages",
+        )?;
+        let config = messages_provider(provider.provider, provider.model)
+            .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?;
+        let model = provider.model.to_owned();
+        let interceptors = ();
+        let context = CallContext::new(&interceptors, litellm_inference::CallOptions::default());
+        provider_call(
+            &AuthServices::default(),
+            config,
+            model,
+            call,
+            secrets,
+            &context,
+        )
+        .await
+    }
+
+    async fn wire_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
+        provider_call_for(MessagesCall {
+            body: body(fields),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://anthropic.test".into()),
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        })
+        .await
+        .map(|call| call.wire.body)
+    }
+
+    #[rstest]
+    #[case::api_key(
+        &[("ANTHROPIC_API_KEY", "sk-secret")],
+        &[("x-api-key", "sk-secret")],
+        "https://api.anthropic.com/v1/messages"
+    )]
+    #[case::auth_token(
+        &[("ANTHROPIC_AUTH_TOKEN", "token")],
+        &[("authorization", "Bearer token")],
+        "https://api.anthropic.com/v1/messages"
+    )]
+    #[case::api_base(
+        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
+        &[("x-api-key", "sk-secret")],
+        "https://gateway.test/v1/messages"
+    )]
+    #[case::sdk_base_url(
+        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
+        &[("x-api-key", "sk-secret")],
+        "https://sdk.test/v1/messages"
+    )]
+    #[tokio::test]
+    async fn credentials_and_base_come_from_the_resolved_secrets(
+        shaping: MessagesShaping,
+        #[case] secrets: &[(&str, &str)],
+        #[case] expected_auth: &[(&str, &str)],
+        #[case] expected_url: &str,
+    ) {
+        let lookup = |name: &str| {
+            secrets
+                .iter()
+                .find(|(key, _)| *key == name)
+                .map(|(_, value)| value.to_string())
+        };
+        let call = provider_call_with_secrets(
+            MessagesCall {
+                body: body(
+                    json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
+                ),
+                api_key: None,
+                api_base: None,
+                custom_llm_provider: Some("anthropic".into()),
+                litellm_params: Default::default(),
+                extra_headers: None,
+                provider_specific_header: None,
+                timeout: None,
+                shaping,
+            },
+            &lookup,
+        )
+        .await
+        .unwrap();
+        let auth: Vec<(&str, &str)> = call
+            .wire
+            .headers
+            .iter()
+            .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
+            .map(|(name, value)| (name.as_str(), value.as_str()))
+            .collect();
+        assert_eq!(
+            (auth.as_slice(), call.wire.url.as_str()),
+            (expected_auth, expected_url)
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn missing_credentials_are_reported_before_the_request_is_transformed(
+        shaping: MessagesShaping,
+    ) {
+        let result = provider_call_for(MessagesCall {
+            body: body(
+                json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}]}),
+            ),
+            api_key: None,
+            api_base: None,
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        })
+        .await;
+        assert!(
+            matches!(
+                result,
+                Err(Error::Auth(litellm_auth::Error::MissingApiKey { .. }))
+            ),
+            "{:?}",
+            result.err()
+        );
+    }
+
+    #[rstest]
+    #[case::nothing_configured(None, None, None)]
+    #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
+    #[case::call_timeout_wins(
+        Some(Duration::from_secs(30)),
+        Some(1800.0),
+        Some(Duration::from_secs(30))
+    )]
+    #[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
+    #[case::unusable_setting_ignored(None, Some(-1.0), None)]
+    #[case::zero_setting_ignored(None, Some(0.0), None)]
+    #[tokio::test]
+    async fn timeout_falls_back_to_the_configured_request_timeout(
+        shaping: MessagesShaping,
+        #[case] call_timeout: Option<Duration>,
+        #[case] request_timeout: Option<f64>,
+        #[case] expected: Option<Duration>,
+    ) {
+        let call = provider_call_for(MessagesCall {
+            body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://anthropic.test".into()),
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: call_timeout,
+            shaping: MessagesShaping {
+                settings: MessagesSettings { request_timeout, ..shaping.settings },
+                ..shaping
+            },
+        })
+        .await
+        .unwrap();
+        assert_eq!(call.timeout, expected);
+    }
+
+    #[rstest]
+    #[case::top_level_and_nested_paths(
+        json!({
+            "max_tokens": 1024,
+            "thinking": {"type": "enabled", "budget_tokens": 2048},
+            "context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
+            "metadata": {"user_id": "u1"},
+            "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
+        }),
+        &["thinking", "context_management", "tools[*].input_examples"],
+        json!({
+            "max_tokens": 1024,
+            "metadata": {"user_id": "u1"},
+            "tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
+        }),
+    )]
+    #[case::no_paths(
+        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[],
+        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
+    )]
+    #[case::model_and_messages_are_never_dropped(
+        json!({"max_tokens": 16}),
+        &["model", "messages", "messages[0].content"],
+        json!({"max_tokens": 16}),
+    )]
+    #[tokio::test]
+    async fn wire_body_drops_configured_paths(
+        shaping: MessagesShaping,
+        #[case] fields: Value,
+        #[case] additional_drop_params: &[&str],
+        #[case] expected_fields: Value,
+    ) {
+        let with_messages = |fields: Value| -> Value {
+            let Value::Object(fields) = fields else {
+                unreachable!()
+            };
+            Value::Object(
+                [
+                    ("model".to_string(), json!("claude-test")),
+                    (
+                        "messages".to_string(),
+                        json!([{"role": "user", "content": "hi"}]),
+                    ),
+                ]
+                .into_iter()
+                .chain(fields)
+                .collect(),
+            )
+        };
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                additional_drop_params: additional_drop_params
+                    .iter()
+                    .map(ToString::to_string)
+                    .collect(),
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert_eq!(
+            wire_body(with_messages(fields), shaping).await,
+            Ok(with_messages(expected_fields))
+        );
+    }
+
+    #[rstest]
+    #[case::model_prefix_picks_the_provider(
+        "azure_ai/claude-test",
+        None,
+        &[("x-priority", "extra"), ("x-scoped", "azure_ai")]
+    )]
+    #[case::explicit_provider(
+        "claude-test",
+        Some("anthropic"),
+        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
+    )]
+    #[case::provider_prefix_on_an_anthropic_model(
+        "anthropic/claude-test",
+        None,
+        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
+    )]
+    #[tokio::test]
+    async fn provider_specific_headers_follow_the_resolved_provider(
+        shaping: MessagesShaping,
+        #[case] model: &str,
+        #[case] custom_llm_provider: Option<&str>,
+        #[case] expected: &[(&str, &str)],
+    ) {
+        let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
+            {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
+            {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
+        ]))
+        .unwrap();
+        let call = provider_call_for(MessagesCall {
+            body: body(
+                json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
+            ),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://resource.services.ai.azure.com".into()),
+            custom_llm_provider: custom_llm_provider.map(Into::into),
+            litellm_params: Default::default(),
+            extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
+            provider_specific_header: Some(configured),
+            timeout: None,
+            shaping,
+        })
+        .await
+        .unwrap();
+        let caller_headers: Vec<(&str, &str)> = call
+            .wire
+            .headers
+            .iter()
+            .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
+            .map(|(name, value)| (name.as_str(), value.as_str()))
+            .collect();
+        assert_eq!(caller_headers, expected);
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn wire_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "anthropic/claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16
+                }),
+                shaping,
+            )
+            .await,
+            Ok(json!({
+                "model": "claude-test",
+                "messages": [{"role": "user", "content": "hi"}],
+                "max_tokens": 16
+            }))
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                reasoning_auto_summary: true,
+                additional_drop_params: vec!["thinking.display".to_string()],
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 4096,
+                    "thinking": {"type": "enabled", "budget_tokens": 2048}
+                }),
+                shaping,
+            )
+            .await,
+            Ok(json!({
+                "model": "claude-test",
+                "messages": [{"role": "user", "content": "hi"}],
+                "max_tokens": 4096,
+                "thinking": {"type": "enabled", "budget_tokens": 2048}
+            }))
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(
+        shaping: MessagesShaping,
+    ) {
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                additional_drop_params: vec!["metadata.user_id".to_string()],
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert!(matches!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16,
+                    "metadata": {"user_id": 123}
+                }),
+                shaping,
+            )
+            .await,
+            Err(Error::InvalidRequest(_))
+        ));
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn wire_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16,
+                    "metadata": {"user_id": 123}
+                }),
+                shaping,
+            )
+            .await,
+            Err(Error::InvalidRequest(
+                litellm_llms::ErrorDetail::InvalidValue {
+                    field: "metadata.user_id",
+                    expected: "a string",
+                    actual: json!(123),
+                }
+            ))
+        );
+    }
+
+    #[rstest]
+    #[case::mantle(
+        "bedrock_mantle/claude-test",
+        "https://bedrock-mantle.us-west-2.api.aws",
+        "/anthropic/v1/messages",
+        "bedrock"
+    )]
+    #[case::platform(
+        "bedrock/claude_platform/claude-test",
+        "https://aws-external-anthropic.us-west-2.api.aws",
+        "/v1/messages",
+        "aws-external-anthropic"
+    )]
+    #[tokio::test]
+    async fn explicit_endpoint_region_is_used_by_the_signer(
+        shaping: MessagesShaping,
+        #[case] model: &str,
+        #[case] base: &str,
+        #[case] path: &str,
+        #[case] service: &str,
+    ) {
+        let call = provider_call_with_secrets(MessagesCall {
+            body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
+            api_key: None,
+            api_base: Some(base.into()),
+            custom_llm_provider: None,
+            litellm_params: crate::litellm_params(Map::from_iter([
+                ("model".into(), json!(model)),
+                ("aws_access_key_id".into(), json!("AKIDEXAMPLE")),
+                ("aws_secret_access_key".into(), json!("secret")),
+            ]))
+            .unwrap(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        }, &|name: &str| match name {
+            "AWS_REGION_NAME" => Some("us-east-1".into()),
+            "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
+            _ => None,
+        })
+        .await
+        .unwrap();
+        assert_eq!(call.wire.url, format!("{base}{path}"));
+        let signed = outbound_request(
+            Authenticated {
+                headers: call.wire.headers,
+                signer: call.signer,
+            },
+            call.wire.url,
+            &call.wire.body,
+            None,
+        )
+        .unwrap();
+        let authorization = signed.header("authorization").unwrap_or_default();
+        assert!(
+            authorization.contains(&format!("/us-west-2/{service}/aws4_request")),
+            "{authorization}"
+        );
+    }
 
     #[rstest]
     #[case::event(

litellm-rust/crates/inference-messages/src/lib.rs

@@ -1,18 +1,19 @@
 mod common_utils;
 mod constants;
 mod handler;
-mod prepare;
 pub mod route;
 mod types;
 
 use futures_util::FutureExt;
 use litellm_auth::AuthServices;
-use litellm_host::interceptors::{Cost, ExecutionFacts, Interceptors, ResultSource};
-
-use litellm_inference::{caching::CallCache, context::CallContext};
+use litellm_host::interceptors::Interceptors;
+use litellm_inference::context::CallContext;
+use litellm_inference::provider::resolve_llm_provider;
 use litellm_secrets::source::SecretSource;
 use std::sync::Arc;
 
+use crate::common_utils::messages_provider;
+
 pub use litellm_inference::RouteError as Error;
 pub use types::{
     MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
@@ -74,39 +75,23 @@ impl MessagesRoute {
         context: CallContext<'_, impl Interceptors<Error>>,
     ) -> Result<MessagesCallResponse, Error> {
         litellm_inference::diagnostic::call(async {
-            let prepared = prepare::prepare(call, self.secrets.as_ref()).await?;
+            let provider = resolve_llm_provider(
+                &call.body.model,
+                call.custom_llm_provider.as_deref(),
+                "messages",
+            )?;
+            let config = messages_provider(provider.provider, provider.model).ok_or_else(|| {
+                Error::InvalidProvider(<&str>::from(provider.provider).to_string())
+            })?;
             litellm_inference::diagnostic::provider(
-                &prepared.body.model,
-                prepared.provider.as_str(),
+                provider.model,
+                <&'static str>::from(provider.provider),
             );
-            let request = self.prepare_outbound(prepared, &context).boxed().await?;
-            let cache = CallCache::<route::Messages>::from_wire(
-                self.cache.as_ref().filter(|_| request.cacheable()),
-                context.cache,
-                &request.identity,
-                &request.wire,
-            );
-            let identity = request.identity.clone();
-            let config = request.provider.config();
-            let (output, source) = match cache.lookup().await {
-                Some(hit) => hit,
-                None => (
-                    self.call_provider(request, &context).await?,
-                    ResultSource::Provider,
-                ),
-            };
-            let reported = match &output {
-                MessagesCallResponse::Complete(response) => config.reported_cost(response),
-                MessagesCallResponse::Stream { .. } => None,
-            };
-            context
-                .result_ready(ExecutionFacts {
-                    provider: identity,
-                    cost: Cost::settle(&source, reported),
-                    source: source.clone(),
-                })
-                .await?;
-            Ok(cache.finish(output, &source).await)
+            let model = provider.model.to_owned();
+            let secrets = self.secrets.resolve(config.config().secret_names()).await?;
+            handler::execute(self, config, model, call, secrets, &context)
+                .boxed()
+                .await
         })
         .await
     }

litellm-rust/crates/inference-messages/src/prepare.rs

@@ -1,554 +0,0 @@
-use std::time::Duration;
-
-use litellm_auth::SecretValue;
-use litellm_core_utils::{
-    dot_notation_indexing::delete_nested_value,
-    get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
-};
-use litellm_http::request::with_default_headers;
-use litellm_llms::base_llm::{
-    auth::ValidatedEnvironment, messages::context::MessagesTransformContext,
-};
-use litellm_llms_types::formats::messages::MessagesRequest;
-use litellm_secrets::source::SecretSource;
-
-use super::{
-    Error, MessagesCall,
-    common_utils::{MessagesProvider, messages_provider, string_headers},
-    types::invalid_request,
-};
-use litellm_inference::provider::resolve_llm_provider;
-
-struct ResolvedProvider {
-    model: String,
-    provider: MessagesProvider,
-}
-
-pub(super) struct ProviderMessagesRequest {
-    pub(super) provider: MessagesProvider,
-    pub(super) url: String,
-    pub(super) body: MessagesRequest,
-    pub(super) environment: ValidatedEnvironment,
-    pub(super) timeout: Option<Duration>,
-    /// The caller's own credential, reported to the host beside the wire request.
-    pub(super) api_key: Option<SecretValue>,
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) async fn prepare(
-    call: MessagesCall,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderMessagesRequest, Error> {
-    let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
-    let secrets = secrets
-        .resolve(resolved.provider.config().secret_names())
-        .await?;
-    prepare_provider_request(call, resolved, secrets.as_ref())
-}
-
-fn resolve_provider(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<ResolvedProvider, Error> {
-    let resolved = resolve_llm_provider(model, custom_llm_provider, "messages")?;
-    let provider = messages_provider(resolved.provider, resolved.model)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(resolved.provider).to_string()))?;
-    Ok(ResolvedProvider {
-        model: resolved.model.to_string(),
-        provider,
-    })
-}
-
-fn prepare_provider_request(
-    call: MessagesCall,
-    resolved: ResolvedProvider,
-    secrets: &dyn Lookup,
-) -> Result<ProviderMessagesRequest, Error> {
-    let ResolvedProvider { model, provider } = resolved;
-    let MessagesCall {
-        body,
-        api_key,
-        api_base,
-        litellm_params,
-        extra_headers,
-        provider_specific_header,
-        timeout,
-        shaping,
-        ..
-    } = call;
-    let connection_params = litellm_router_types::LitellmParams {
-        api_base: api_base.clone().or(litellm_params.api_base.clone()),
-        ..litellm_params
-    };
-    let config = provider.config();
-    let env_lookup = |key: &str| secrets.get(key);
-
-    let sanitized = config.shape_request(
-        MessagesRequest { model, ..body },
-        shaping.settings.reasoning_auto_summary,
-    )?;
-    let trimmed =
-        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
-    let transformed = config.transform_anthropic_messages_request(
-        trimmed,
-        &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
-    )?;
-
-    let scoped =
-        get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
-    let forwarded = string_headers(Some(
-        extra_headers.into_iter().flatten().chain(scoped).collect(),
-    ))?;
-    let forwarded = config.request_headers(
-        with_default_headers(forwarded, config.default_headers()),
-        &transformed,
-    );
-    let environment = config.validate_environment(
-        forwarded,
-        api_key.as_deref(),
-        &transformed.model,
-        &connection_params,
-        &env_lookup,
-    )?;
-
-    let url = config.get_complete_url(
-        api_base.as_deref(),
-        &transformed.model,
-        &connection_params,
-        transformed.params.stream == Some(true),
-        &env_lookup,
-    )?;
-
-    Ok(ProviderMessagesRequest {
-        provider,
-        url,
-        body: transformed,
-        environment,
-        timeout: timeout.or_else(|| shaping.settings.request_timeout()),
-        api_key: api_key.map(SecretValue::new),
-    })
-}
-
-fn without_additional_drop_params(
-    request: MessagesRequest,
-    paths: &[String],
-) -> Result<MessagesRequest, Error> {
-    if paths.is_empty() {
-        return Ok(request);
-    }
-    let params = serde_json::to_value(request.params).map_err(invalid_request)?;
-    let trimmed = paths
-        .iter()
-        .fold(params, |params, path| delete_nested_value(params, path));
-    Ok(MessagesRequest {
-        params: serde_json::from_value(trimmed).map_err(invalid_request)?,
-        ..request
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use litellm_llms::base_llm::auth::resolve_auth;
-    use litellm_llms_types::headers::ProviderSpecificHeaders;
-    use rstest::{fixture, rstest};
-    use serde_json::{Map, Value, json};
-
-    use super::*;
-    use crate::{MessagesSettings, MessagesShaping};
-
-    #[fixture]
-    fn shaping() -> MessagesShaping {
-        MessagesShaping::default()
-    }
-
-    fn body(value: Value) -> MessagesRequest {
-        serde_json::from_value(value).unwrap()
-    }
-
-    fn prepare(call: MessagesCall) -> Result<ProviderMessagesRequest, Error> {
-        prepare_with_secrets(call, &|_: &str| None)
-    }
-
-    fn prepare_with_secrets(
-        call: MessagesCall,
-        secrets: &dyn Lookup,
-    ) -> Result<ProviderMessagesRequest, Error> {
-        let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
-        prepare_provider_request(call, resolved, secrets)
-    }
-
-    /// The headers as they go on the wire, credential applied.
-    fn wire_headers(prepared: &ProviderMessagesRequest) -> Vec<(String, String)> {
-        tokio::runtime::Builder::new_current_thread()
-            .build()
-            .unwrap()
-            .block_on(resolve_auth(
-                &litellm_auth::AuthServices::default(),
-                prepared.environment.clone(),
-                &|_| None,
-            ))
-            .unwrap()
-            .headers
-    }
-
-    #[rstest]
-    #[case::api_key(
-        &[("ANTHROPIC_API_KEY", "sk-secret")],
-        &[("x-api-key", "sk-secret")],
-        "https://api.anthropic.com/v1/messages"
-    )]
-    #[case::auth_token(
-        &[("ANTHROPIC_AUTH_TOKEN", "token")],
-        &[("authorization", "Bearer token")],
-        "https://api.anthropic.com/v1/messages"
-    )]
-    #[case::api_base(
-        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
-        &[("x-api-key", "sk-secret")],
-        "https://gateway.test/v1/messages"
-    )]
-    #[case::sdk_base_url(
-        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
-        &[("x-api-key", "sk-secret")],
-        "https://sdk.test/v1/messages"
-    )]
-    fn credentials_and_base_come_from_the_resolved_secrets(
-        shaping: MessagesShaping,
-        #[case] secrets: &[(&str, &str)],
-        #[case] expected_auth: &[(&str, &str)],
-        #[case] expected_url: &str,
-    ) {
-        let lookup = |name: &str| {
-            secrets
-                .iter()
-                .find(|(key, _)| *key == name)
-                .map(|(_, value)| value.to_string())
-        };
-        let prepared = prepare_with_secrets(
-            MessagesCall {
-                body: body(
-                    json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
-                ),
-                api_key: None,
-                api_base: None,
-                custom_llm_provider: Some("anthropic".into()),
-                litellm_params: Default::default(),
-                extra_headers: None,
-                provider_specific_header: None,
-                timeout: None,
-                shaping,
-            },
-            &lookup,
-        )
-        .unwrap();
-        let headers = wire_headers(&prepared);
-        let auth: Vec<(&str, &str)> = headers
-            .iter()
-            .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
-            .map(|(name, value)| (name.as_str(), value.as_str()))
-            .collect();
-        assert_eq!(
-            (auth.as_slice(), prepared.url.as_str()),
-            (expected_auth, expected_url)
-        );
-    }
-
-    #[rstest]
-    #[case::nothing_configured(None, None, None)]
-    #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
-    #[case::call_timeout_wins(
-        Some(Duration::from_secs(30)),
-        Some(1800.0),
-        Some(Duration::from_secs(30))
-    )]
-    #[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
-    #[case::unusable_setting_ignored(None, Some(-1.0), None)]
-    #[case::zero_setting_ignored(None, Some(0.0), None)]
-    fn timeout_falls_back_to_the_configured_request_timeout(
-        shaping: MessagesShaping,
-        #[case] call_timeout: Option<Duration>,
-        #[case] request_timeout: Option<f64>,
-        #[case] expected: Option<Duration>,
-    ) {
-        let prepared = prepare(MessagesCall {
-            body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://anthropic.test".into()),
-            custom_llm_provider: Some("anthropic".into()),
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: call_timeout,
-            shaping: MessagesShaping {
-                settings: MessagesSettings { request_timeout, ..shaping.settings },
-                ..shaping
-            },
-        })
-        .unwrap();
-        assert_eq!(prepared.timeout, expected);
-    }
-
-    fn prepared_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
-        prepare(MessagesCall {
-            body: body(fields),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://anthropic.test".into()),
-            custom_llm_provider: Some("anthropic".into()),
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: None,
-            shaping,
-        })
-        .map(|prepared| serde_json::to_value(prepared.body).unwrap())
-    }
-
-    #[rstest]
-    #[case::top_level_and_nested_paths(
-        json!({
-            "max_tokens": 1024,
-            "thinking": {"type": "enabled", "budget_tokens": 2048},
-            "context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
-            "metadata": {"user_id": "u1"},
-            "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
-        }),
-        &["thinking", "context_management", "tools[*].input_examples"],
-        json!({
-            "max_tokens": 1024,
-            "metadata": {"user_id": "u1"},
-            "tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
-        }),
-    )]
-    #[case::no_paths(
-        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
-        &[],
-        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
-    )]
-    #[case::model_and_messages_are_never_dropped(
-        json!({"max_tokens": 16}),
-        &["model", "messages", "messages[0].content"],
-        json!({"max_tokens": 16}),
-    )]
-    fn prepared_body_drops_configured_paths(
-        shaping: MessagesShaping,
-        #[case] fields: Value,
-        #[case] additional_drop_params: &[&str],
-        #[case] expected_fields: Value,
-    ) {
-        let with_messages = |fields: Value| -> Value {
-            let Value::Object(fields) = fields else {
-                unreachable!()
-            };
-            Value::Object(
-                [
-                    ("model".to_string(), json!("claude-test")),
-                    (
-                        "messages".to_string(),
-                        json!([{"role": "user", "content": "hi"}]),
-                    ),
-                ]
-                .into_iter()
-                .chain(fields)
-                .collect(),
-            )
-        };
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                additional_drop_params: additional_drop_params
-                    .iter()
-                    .map(ToString::to_string)
-                    .collect(),
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert_eq!(
-            prepared_body(with_messages(fields), shaping),
-            Ok(with_messages(expected_fields))
-        );
-    }
-
-    #[rstest]
-    #[case::model_prefix_picks_the_provider(
-        "azure_ai/claude-test",
-        None,
-        &[("x-priority", "extra"), ("x-scoped", "azure_ai")]
-    )]
-    #[case::explicit_provider(
-        "claude-test",
-        Some("anthropic"),
-        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
-    )]
-    #[case::provider_prefix_on_an_anthropic_model(
-        "anthropic/claude-test",
-        None,
-        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
-    )]
-    fn provider_specific_headers_follow_the_resolved_provider(
-        shaping: MessagesShaping,
-        #[case] model: &str,
-        #[case] custom_llm_provider: Option<&str>,
-        #[case] expected: &[(&str, &str)],
-    ) {
-        let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
-            {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
-            {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
-        ]))
-        .unwrap();
-        let prepared = prepare(MessagesCall {
-            body: body(
-                json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
-            ),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://resource.services.ai.azure.com".into()),
-            custom_llm_provider: custom_llm_provider.map(Into::into),
-            litellm_params: Default::default(),
-            extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
-            provider_specific_header: Some(configured),
-            timeout: None,
-            shaping,
-        })
-        .unwrap();
-        let caller_headers: Vec<(&str, &str)> = prepared
-            .environment
-            .headers
-            .iter()
-            .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
-            .map(|(name, value)| (name.as_str(), value.as_str()))
-            .collect();
-        assert_eq!(caller_headers, expected);
-    }
-
-    #[rstest]
-    fn prepared_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "anthropic/claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16
-                }),
-                shaping,
-            ),
-            Ok(json!({
-                "model": "claude-test",
-                "messages": [{"role": "user", "content": "hi"}],
-                "max_tokens": 16
-            }))
-        );
-    }
-
-    #[rstest]
-    fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                reasoning_auto_summary: true,
-                additional_drop_params: vec!["thinking.display".to_string()],
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 4096,
-                    "thinking": {"type": "enabled", "budget_tokens": 2048}
-                }),
-                shaping,
-            ),
-            Ok(json!({
-                "model": "claude-test",
-                "messages": [{"role": "user", "content": "hi"}],
-                "max_tokens": 4096,
-                "thinking": {"type": "enabled", "budget_tokens": 2048}
-            }))
-        );
-    }
-
-    #[rstest]
-    fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(shaping: MessagesShaping) {
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                additional_drop_params: vec!["metadata.user_id".to_string()],
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert!(matches!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16,
-                    "metadata": {"user_id": 123}
-                }),
-                shaping,
-            ),
-            Err(Error::InvalidRequest(_))
-        ));
-    }
-
-    #[rstest]
-    fn prepared_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16,
-                    "metadata": {"user_id": 123}
-                }),
-                shaping,
-            ),
-            Err(Error::InvalidRequest(
-                litellm_llms::ErrorDetail::InvalidValue {
-                    field: "metadata.user_id",
-                    expected: "a string",
-                    actual: json!(123),
-                }
-            ))
-        );
-    }
-    #[rstest]
-    #[case::mantle(
-        "bedrock_mantle/claude-test",
-        "https://bedrock-mantle.us-west-2.api.aws",
-        "/anthropic/v1/messages",
-        "bedrock"
-    )]
-    #[case::platform(
-        "bedrock/claude_platform/claude-test",
-        "https://aws-external-anthropic.us-west-2.api.aws",
-        "/v1/messages",
-        "aws-external-anthropic"
-    )]
-    fn explicit_endpoint_region_is_used_by_the_signer(
-        shaping: MessagesShaping,
-        #[case] model: &str,
-        #[case] base: &str,
-        #[case] path: &str,
-        #[case] service: &str,
-    ) {
-        let prepared = prepare_with_secrets(MessagesCall {
-            body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
-            api_key: None,
-            api_base: Some(base.into()),
-            custom_llm_provider: None,
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: None,
-            shaping,
-        }, &|name: &str| match name {
-            "AWS_REGION_NAME" => Some("us-east-1".into()),
-            "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
-            _ => None,
-        }).unwrap();
-        assert_eq!(prepared.url, format!("{base}{path}"));
-        assert!(
-            matches!(prepared.environment.auth, litellm_llms::base_llm::auth::AuthScheme::AwsSigV4 { region, service: actual, .. } if region == "us-west-2" && actual == service)
-        );
-    }
-}

litellm-rust/crates/inference-messages/tests/messages/request.rs

@@ -343,6 +343,77 @@ async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
     assert_eq!(sent, expected);
 }
 
+#[rstest]
+#[case::anthropic("anthropic", &[AnthropicBeta::DangerousToolUse20260903])]
+#[case::deepseek("deepseek", &[])]
+#[case::minimax("minimax", &[])]
+#[tokio::test]
+async fn safeguards_add_the_dangerous_tool_use_beta_only_where_the_beta_map_has_a_column(
+    call: MessagesCall,
+    #[case] provider: &str,
+    #[case] derived: &[AnthropicBeta],
+) {
+    let upstream = upstream([message_response()]).await;
+    let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
+
+    run_message(with_fields(
+        MessagesCall {
+            custom_llm_provider: Some(provider.into()),
+            api_key: Some("sk".into()),
+            api_base: Some(upstream.uri()),
+            extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
+            ..call
+        },
+        json!({"safeguards": safeguards}),
+    ))
+    .await;
+
+    let request = only_request(&upstream).await;
+    let expected: BetaSet = derived
+        .iter()
+        .cloned()
+        .chain([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
+        .collect();
+    assert_eq!(sent_betas(&request), expected);
+    assert_eq!(
+        request.body_json::<Value>().unwrap()["safeguards"],
+        safeguards
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn null_safeguards_are_dropped_instead_of_sent_without_their_beta(
+    call: MessagesCall,
+    #[values("anthropic", "azure_ai", "bedrock_mantle", "deepseek", "minimax")] provider: &str,
+) {
+    let upstream = upstream([message_response()]).await;
+
+    run_message(with_fields(
+        MessagesCall {
+            custom_llm_provider: Some(provider.into()),
+            api_key: Some("sk".into()),
+            api_base: Some(upstream.uri()),
+            extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
+            ..call
+        },
+        json!({"safeguards": null}),
+    ))
+    .await;
+
+    let request = only_request(&upstream).await;
+    assert_eq!(
+        sent_betas(&request),
+        BetaSet::from_iter([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
+    );
+    assert!(
+        !request
+            .body_json::<Map<String, Value>>()
+            .unwrap()
+            .contains_key("safeguards")
+    );
+}
+
 #[rstest]
 #[tokio::test]
 async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: MessagesCall) {

litellm-rust/crates/inference-ocr/src/arguments.rs

@@ -44,8 +44,8 @@ pub fn consumed_optional_param_names(
     model: &str,
     custom_llm_provider: Option<&str>,
 ) -> Result<Vec<&'static str>, Error> {
-    let (model, config) = resolve_provider_config(model, custom_llm_provider)?;
-    let provider_fields = config.get_supported_ocr_params(&model);
+    let (provider, config) = resolve_provider_config(model, custom_llm_provider)?;
+    let provider_fields = config.get_supported_ocr_params(provider.model);
     let auth_fields: &[&str] = match config {
         OcrConfigKind::AwsTextract | OcrConfigKind::AwsTextractAnalyze => AWS_AUTH_OPTION_FIELDS,
         OcrConfigKind::AzureAi

litellm-rust/crates/inference-ocr/src/client.rs

@@ -1,81 +0,0 @@
-use litellm_host::observation::ObservationSender;
-use std::sync::Arc;
-
-use litellm_host::interceptors::Interceptors;
-use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
-use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
-
-use super::{
-    handler::perform_ocr_request,
-    types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest},
-};
-
-#[derive(Clone)]
-pub struct OcrRoute {
-    client: OcrClient,
-}
-
-impl OcrRoute {
-    pub fn new(client: OcrClient) -> Self {
-        Self { client }
-    }
-
-    pub async fn execute(
-        &self,
-        request: LiteLLMOcrRequest,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<ObservationSender>,
-    ) -> Result<LiteLLMOcrResponse, Error> {
-        litellm_host::lifecycle::observe_unary(
-            observers.clone(),
-            self.run(request, interceptors, observers.as_ref()),
-        )
-        .await
-    }
-
-    #[tracing::instrument(name = "litellm.route", skip_all, fields(
-        route = "ocr",
-        model = %request.model,
-        resolved_model = %request.model,
-        provider = <&str>::from(request.config.provider()),
-        stream = false,
-        outcome
-    ))]
-    pub(super) async fn run(
-        &self,
-        request: LiteLLMOcrRequest,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<LiteLLMOcrResponse, Error> {
-        litellm_inference::diagnostic::unary(async {
-            let caller_document = matches!(&request.document, OcrDocumentInput::Document(_));
-            let prepared = prepare_request_document(request).await?;
-            let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
-                Box::pin(perform_ocr_request(
-                    &self.client,
-                    prepared,
-                    interceptors,
-                    caller_document,
-                    observers,
-                ));
-            execute.await
-        })
-        .await
-    }
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-async fn prepare_request_document(
-    request: LiteLLMOcrRequest<OcrDocumentInput>,
-) -> Result<ResolvedOcrRequest, Error> {
-    if let OcrDocumentInput::Document(_) = &request.document {
-        return request.map_document(super::document::prepare_document);
-    }
-    let logger = litellm_tracing::Logger::current();
-    let span = tracing::Span::current();
-    tokio::task::spawn_blocking(move || {
-        logger.scope(|| span.in_scope(|| request.map_document(super::document::prepare_document)))
-    })
-    .await
-    .map_err(|error| Error::DocumentTask(Arc::new(error)))?
-}

litellm-rust/crates/inference-ocr/src/document.rs

@@ -1,10 +1,10 @@
-use std::{collections::BTreeMap as Map, io::Read, path::Path};
+use std::{collections::BTreeMap as Map, io::Read, path::Path, sync::Arc};
 
 use base64::{Engine, engine::general_purpose::STANDARD};
 use litellm_llms::base_llm::ocr::{error::Error, transformation::OCR_INLINE_MAX_BYTES};
 use litellm_llms_types::formats::ocr::OcrDocument;
 
-use crate::types::OcrDocumentInput;
+use crate::types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest};
 
 pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
     match input {
@@ -24,6 +24,21 @@ pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
     }
 }
 
+pub(crate) async fn resolve_document(
+    request: LiteLLMOcrRequest<OcrDocumentInput>,
+) -> Result<ResolvedOcrRequest, Error> {
+    if let OcrDocumentInput::Document(_) = &request.document {
+        return request.map_document(prepare_document);
+    }
+    let logger = litellm_tracing::Logger::current();
+    let span = tracing::Span::current();
+    tokio::task::spawn_blocking(move || {
+        logger.scope(|| span.in_scope(|| request.map_document(prepare_document)))
+    })
+    .await
+    .map_err(|error| Error::DocumentTask(Arc::new(error)))?
+}
+
 pub fn read_path_document(path: &Path, mime_type: Option<&str>) -> Result<OcrDocument, Error> {
     let mut bytes = Vec::new();
     std::fs::File::open(path)

litellm-rust/crates/inference-ocr/src/handler.rs

@@ -4,29 +4,45 @@ use litellm_host::{lifecycle::ExecutionEvent, observation::ObservationSender};
 use litellm_llms::base_llm::ocr::{
     error::Error,
     handler::{CallHooks, OcrClient},
-    transformation::PreparedOcrRequest,
+    transformation::{OcrConnection, PreparedOcrRequest},
 };
 use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
-use super::{arguments::is_secret_param, prepare::prepare_request, provider_config::OcrConfigKind};
-use crate::types::ResolvedOcrRequest;
+use super::{arguments::is_secret_param, provider_config::OcrConfigKind};
+use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
 
-pub(crate) async fn perform_ocr_request(
+pub(crate) async fn execute(
     client: &OcrClient,
-    request: ResolvedOcrRequest,
+    config: OcrConfigKind,
+    call: ResolvedOcrRequest,
+    secrets: Secrets,
     host: &impl Interceptors<Error>,
     caller_document: bool,
     observers: Option<&ObservationSender>,
 ) -> Result<LiteLLMOcrResponse, Error> {
-    request.response_format()?;
-    let config = request.config;
-    let secrets = client
-        .secret_source()
-        .resolve(&config.secret_names())
-        .await
-        .map_err(|error| Error::Secret(std::sync::Arc::new(error)))?;
-    let request = prepare_request(request, caller_document, client, secrets);
+    call.response_format()?;
+    let LiteLLMOcrRequest {
+        model,
+        document,
+        credentials,
+        transport,
+        optional_params,
+        input_sources,
+        azure_ad_token_provider,
+        ..
+    } = call;
+    let resolved = config.resolve_credentials(credentials, &secrets);
+    let request = PreparedOcrRequest {
+        model,
+        document,
+        connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
+        caller_document,
+        optional_params,
+        input_sources,
+        azure_ad_token_provider,
+    };
     let interceptors = OcrCallHooks::new(host, &request, config, observers);
     config.ocr(client, &request, &interceptors).await
 }
@@ -86,3 +102,257 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
         Box::pin(self.interceptors.after_provider_response(raw))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
+    use litellm_llms_types::formats::ocr::OcrResponseFormat;
+    use rstest::rstest;
+    use serde_json::{Value, json};
+    use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
+
+    use super::*;
+    use crate::{
+        OcrRoute,
+        types::LiteLLMOcrRequest,
+        wire::{OcrWireRequest, decode_request},
+    };
+
+    fn client() -> OcrClient {
+        OcrClient::for_test(
+            litellm_http::Client::plain_for_test(),
+            litellm_http::Client::no_redirect_for_test(),
+        )
+        .with_secrets(litellm_inference_testing::no_secrets())
+    }
+
+    fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
+        decode_request(OcrWireRequest {
+            model: model.into(),
+            document,
+            api_key: Some(litellm_auth::SecretValue::new("test-key")),
+            api_base: Some(base.into()),
+            custom_llm_provider: None,
+            extra_headers: None,
+            optional_params: options.as_object().unwrap().clone(),
+            input_sources: Default::default(),
+            timeout_seconds: Some(2.0),
+        })
+        .unwrap()
+    }
+
+    async fn run_request(request: LiteLLMOcrRequest) -> Result<LiteLLMOcrResponse, Error> {
+        OcrRoute::new(client()).execute(request, &(), None).await
+    }
+
+    async fn upstream(request_count: u64, response: Value) -> MockServer {
+        let server = MockServer::start().await;
+        Mock::given(any())
+            .respond_with(ResponseTemplate::new(200).set_body_json(response))
+            .up_to_n_times(request_count)
+            .mount(&server)
+            .await;
+        server
+    }
+
+    fn image(url: &str) -> Value {
+        json!({"type": "image_url", "image_url": url})
+    }
+
+    fn body(request: &wiremock::Request) -> Value {
+        serde_json::from_slice(&request.body).unwrap()
+    }
+
+    fn assert_mistral_request(request: &wiremock::Request) {
+        assert_eq!(
+            request
+                .headers
+                .get("authorization")
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "Bearer test-key"
+        );
+        assert_eq!(
+            request
+                .headers
+                .get("content-type")
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "application/json"
+        );
+        assert_eq!(
+            body(request),
+            json!({
+                "model": "mistral-ocr-maas",
+                "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
+                "pages": [0, 2],
+                "include_image_base64": true,
+                "unknown": "preserved"
+            })
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
+        let upstream = upstream(
+            1,
+            json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
+        )
+        .await;
+        let request = request(
+            "cohere/parse",
+            &upstream.uri(),
+            image("https://example.com/original.png"),
+            json!({
+                "output_format": "markdown", "timeout": 30,
+                "extra_body": {
+                    "output_format": {"future": true},
+                    "document": {"type": "image_url", "image_url": "https://example.com/a.png",
+                        "provider_options": {"nested": [false, 0, null]}}
+                }
+            }),
+        );
+
+        run_request(request).await.unwrap();
+
+        let [request] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        assert_eq!(
+            body(&request),
+            json!({
+                "model": "parse", "output_format": {"future": true},
+                "document": {"type": "image_url", "image_url": "https://example.com/a.png",
+                    "provider_options": {"nested": [false, 0, null]}}
+            })
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn explicit_null_options_use_defaults_before_http() {
+        let upstream = upstream(
+            1,
+            json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
+        )
+        .await;
+        let request = request(
+            "cohere/parse",
+            &upstream.uri(),
+            image("https://example.com/a.png"),
+            json!({"output_format": null, "req_format": null}),
+        );
+        assert_eq!(
+            request.response_format().unwrap(),
+            OcrResponseFormat::Litellm
+        );
+
+        run_request(request).await.unwrap();
+
+        let [request] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        let body = body(&request);
+        assert_eq!(body["output_format"], "markdown");
+        assert!(body.get("req_format").is_none());
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
+        let upstream = upstream(
+            2,
+            json!({
+                "pages": [{"index": 0, "markdown": "hello"}],
+                "extra": "preserved"
+            }),
+        )
+        .await;
+        let options = json!({
+            "pages": [0, 2],
+            "include_image_base64": true,
+            "vertex_project": "project-1",
+            "vertex_location": "us-central1",
+            "unknown": "preserved"
+        });
+        let document =
+            json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
+        let direct_call = request(
+            "mistral/mistral-ocr-maas",
+            &upstream.uri(),
+            document.clone(),
+            options.clone(),
+        );
+        let vertex_call = request(
+            "vertex_ai/mistral-ocr-maas",
+            &upstream.uri(),
+            document,
+            options,
+        );
+        assert_eq!(direct_call.transport.timeout, Some(Duration::from_secs(2)));
+        assert_eq!(vertex_call.transport.timeout, Some(Duration::from_secs(2)));
+        let direct_response = run_request(direct_call).await.unwrap().into_json();
+        let vertex_response = run_request(vertex_call).await.unwrap().into_json();
+
+        let [direct_http, vertex_http] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        assert_eq!(direct_http.url.path(), "/v1/ocr");
+        assert_eq!(
+            vertex_http.url.path(),
+            "/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
+        );
+        assert_mistral_request(&direct_http);
+        assert_mistral_request(&vertex_http);
+        assert_eq!(direct_response, vertex_response);
+        assert_eq!(direct_response["model"], "mistral-ocr-maas");
+        assert_eq!(direct_response["object"], "ocr");
+        assert_eq!(direct_response["extra"], "preserved");
+    }
+
+    #[derive(serde::Deserialize)]
+    struct KnownParams {
+        pages: Option<Vec<i64>>,
+    }
+
+    #[rstest]
+    fn parsed_provider_params_separates_known_and_extra_params() {
+        let arguments: CallArguments = serde_json::from_value(json!({
+            "pages": [0, 2],
+            "future_ocr_option": true,
+            "extra_body": {"provider_option": "value"}
+        }))
+        .unwrap();
+        let known: KnownParams = parse_options(&arguments).unwrap();
+        assert_eq!(known.pages, Some(vec![0, 2]));
+        assert_eq!(arguments["future_ocr_option"], true);
+        assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
+        assert_eq!(
+            arguments
+                .iter()
+                .filter(|(name, _)| name.as_str() != "pages")
+                .count(),
+            2
+        );
+        assert_eq!(
+            compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
+            json!({
+                "pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
+            })
+        );
+    }
+}

litellm-rust/crates/inference-ocr/src/lib.rs

@@ -1,10 +1,82 @@
 pub mod arguments;
-mod client;
-pub use client::OcrRoute;
 pub mod document;
 pub(crate) mod handler;
-pub(crate) mod prepare;
 pub mod provider_config;
 pub mod route;
 pub mod types;
 pub mod wire;
+
+use std::sync::Arc;
+
+use litellm_host::interceptors::Interceptors;
+use litellm_host::observation::ObservationSender;
+use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
+use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
+
+use crate::{
+    handler::execute,
+    types::{LiteLLMOcrRequest, OcrDocumentInput},
+};
+
+#[derive(Clone)]
+pub struct OcrRoute {
+    client: OcrClient,
+}
+
+impl OcrRoute {
+    pub fn new(client: OcrClient) -> Self {
+        Self { client }
+    }
+
+    pub async fn execute(
+        &self,
+        request: LiteLLMOcrRequest,
+        interceptors: &impl Interceptors<Error>,
+        observers: Option<ObservationSender>,
+    ) -> Result<LiteLLMOcrResponse, Error> {
+        litellm_host::lifecycle::observe_unary(
+            observers.clone(),
+            self.run(request, interceptors, observers.as_ref()),
+        )
+        .await
+    }
+
+    #[tracing::instrument(name = "litellm.route", skip_all, fields(
+        route = "ocr",
+        model = %call.model,
+        resolved_model = %call.model,
+        provider = <&str>::from(call.config.provider()),
+        stream = false,
+        outcome
+    ))]
+    async fn run(
+        &self,
+        call: LiteLLMOcrRequest,
+        interceptors: &impl Interceptors<Error>,
+        observers: Option<&ObservationSender>,
+    ) -> Result<LiteLLMOcrResponse, Error> {
+        litellm_inference::diagnostic::unary(async {
+            let caller_document = matches!(&call.document, OcrDocumentInput::Document(_));
+            let call = document::resolve_document(call).await?;
+            let config = call.config;
+            let secrets = self
+                .client
+                .secret_source()
+                .resolve(&config.secret_names())
+                .await
+                .map_err(|error| Error::Secret(Arc::new(error)))?;
+            let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
+                Box::pin(execute(
+                    &self.client,
+                    config,
+                    call,
+                    secrets,
+                    interceptors,
+                    caller_document,
+                    observers,
+                ));
+            execute.await
+        })
+        .await
+    }
+}

litellm-rust/crates/inference-ocr/src/prepare.rs

@@ -1,309 +0,0 @@
-use litellm_auth::{InputSource, SecretValue, Sourced};
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_llms::base_llm::ocr::{
-    handler::OcrClient,
-    transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
-};
-use litellm_secrets::source::Secrets;
-
-use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
-
-pub(crate) fn prepare_request(
-    request: ResolvedOcrRequest,
-    caller_document: bool,
-    client: &OcrClient,
-    secrets: Secrets,
-) -> PreparedOcrRequest {
-    let credentials = request.credentials.clone();
-    let (preferred_api_key_env, api_base_env) = match request.config.provider() {
-        LlmProviders::Mistral => (
-            Some("MISTRAL_AZURE_API_KEY"),
-            Some("MISTRAL_AZURE_API_BASE"),
-        ),
-        LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
-        _ => (None, None),
-    };
-    let secret = |name: &str| secrets.truthy(name);
-    let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
-        credentials.api_key.clone().or_else(|| {
-            preferred_api_key_env
-                .into_iter()
-                .chain(request.config.get_api_key_env_var())
-                .find_map(secret)
-                .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
-        })
-    });
-    let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
-        credentials.api_base.clone().or_else(|| {
-            api_base_env
-                .and_then(secret)
-                .map(|value| Sourced::new(value, InputSource::Environment))
-        })
-    });
-    let resolved = request
-        .config
-        .resolve_connection_params(OcrCredentialInputs {
-            dynamic_api_key,
-            dynamic_api_base,
-            ..credentials
-        });
-    let LiteLLMOcrRequest {
-        model,
-        document,
-        transport,
-        optional_params,
-        input_sources,
-        azure_ad_token_provider,
-        ..
-    } = request;
-    PreparedOcrRequest {
-        model,
-        document,
-        connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
-        caller_document,
-        optional_params,
-        input_sources,
-        azure_ad_token_provider,
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::time::Duration;
-
-    use futures_util::future::BoxFuture;
-    use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
-    use litellm_host::interceptors::WireRequest;
-    use litellm_llms::{
-        base_llm::ocr::{
-            error::Error,
-            handler::{CallHooks, OcrClient},
-            transformation::BaseOcrConfig,
-        },
-        cohere::ocr::transformation::CohereParseConfig,
-        mistral::ocr::transformation::MistralOcrConfig,
-        vertex_ai::ocr::transformation::VertexAiOcrConfig,
-    };
-    use litellm_llms_types::formats::ocr::OcrResponseFormat;
-    use serde_json::{Value, json};
-
-    use super::*;
-    use crate::{
-        document::prepare_document,
-        types::LiteLLMOcrRequest,
-        wire::{OcrWireRequest, decode_request},
-    };
-
-    /// Stands in for a host with no interceptors registered.
-    struct NoHooks;
-
-    impl CallHooks<Error> for NoHooks {
-        fn before_provider_request(
-            &self,
-            wire: WireRequest,
-        ) -> BoxFuture<'_, Result<WireRequest, Error>> {
-            Box::pin(async move { Ok(wire) })
-        }
-
-        fn response_received<'a>(&'a self, _body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
-            Box::pin(async { Ok(()) })
-        }
-    }
-
-    fn client() -> OcrClient {
-        OcrClient::for_test(
-            litellm_http::Client::plain_for_test(),
-            litellm_http::Client::no_redirect_for_test(),
-        )
-    }
-
-    fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
-        decode_request(OcrWireRequest {
-            model: model.into(),
-            document,
-            api_key: Some(litellm_auth::SecretValue::new("test-key")),
-            api_base: Some(base.into()),
-            custom_llm_provider: None,
-            extra_headers: None,
-            optional_params: options.as_object().unwrap().clone(),
-            input_sources: Default::default(),
-            timeout_seconds: Some(2.0),
-        })
-        .unwrap()
-    }
-
-    fn prepared(request: LiteLLMOcrRequest) -> PreparedOcrRequest {
-        prepare_request(
-            request.map_document(prepare_document).unwrap(),
-            true,
-            &client(),
-            std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
-        )
-    }
-
-    fn image(url: &str) -> Value {
-        json!({"type": "image_url", "image_url": url})
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
-        let request = request(
-            "cohere/parse",
-            "https://example.com",
-            image("https://example.com/original.png"),
-            json!({
-                "output_format": "markdown", "timeout": 30,
-                "extra_body": {
-                    "output_format": {"future": true},
-                    "document": {"type": "image_url", "image_url": "https://example.com/a.png",
-                        "provider_options": {"nested": [false, 0, null]}}
-                }
-            }),
-        );
-
-        let http = CohereParseConfig
-            .prepare_request(&prepared(request), &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        let body: Value = serde_json::from_slice(http.body()).unwrap();
-        assert_eq!(
-            body,
-            json!({
-                "model": "parse", "output_format": {"future": true},
-                "document": {"type": "image_url", "image_url": "https://example.com/a.png",
-                    "provider_options": {"nested": [false, 0, null]}}
-            })
-        );
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn explicit_null_options_use_defaults_before_http() {
-        let request = request(
-            "cohere/parse",
-            "https://example.com",
-            image("https://example.com/a.png"),
-            json!({"output_format": null, "req_format": null}),
-        );
-        assert_eq!(
-            request.response_format().unwrap(),
-            OcrResponseFormat::Litellm
-        );
-
-        let http = CohereParseConfig
-            .prepare_request(&prepared(request), &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        let body: Value = serde_json::from_slice(http.body()).unwrap();
-        assert_eq!(body["output_format"], "markdown");
-        assert!(body.get("req_format").is_none());
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
-        let options = json!({
-            "pages": [0, 2],
-            "include_image_base64": true,
-            "vertex_project": "project-1",
-            "vertex_location": "us-central1",
-            "unknown": "preserved"
-        });
-        let document =
-            json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
-        let direct = prepared(request(
-            "mistral/mistral-ocr-maas",
-            "https://mistral.test",
-            document.clone(),
-            options.clone(),
-        ));
-        let vertex = prepared(request(
-            "vertex_ai/mistral-ocr-maas",
-            "https://vertex.test",
-            document,
-            options,
-        ));
-
-        let direct_http = MistralOcrConfig
-            .prepare_request(&direct, &client(), &NoHooks)
-            .await
-            .unwrap();
-        let vertex_http = VertexAiOcrConfig
-            .prepare_request(&vertex, &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        assert_eq!(direct_http.url(), "https://mistral.test/v1/ocr");
-        assert_eq!(
-            vertex_http.url(),
-            "https://vertex.test/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
-        );
-        for http in [&direct_http, &vertex_http] {
-            assert_eq!(http.header("authorization").unwrap(), "Bearer test-key");
-            assert_eq!(http.header("content-type").unwrap(), "application/json");
-            assert_eq!(http.timeout(), Some(Duration::from_secs(2)));
-            let body: Value = serde_json::from_slice(http.body()).unwrap();
-            assert_eq!(
-                body,
-                json!({
-                    "model": "mistral-ocr-maas",
-                    "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
-                    "pages": [0, 2],
-                    "include_image_base64": true,
-                    "unknown": "preserved"
-                })
-            );
-        }
-        let payload = serde_json::to_vec(
-            &json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}),
-        )
-        .unwrap();
-        let direct_response = MistralOcrConfig
-            .transform_ocr_response(&direct.model, &payload, OcrResponseFormat::Litellm)
-            .unwrap()
-            .into_json();
-        let vertex_response = VertexAiOcrConfig
-            .transform_ocr_response(&vertex.model, &payload, OcrResponseFormat::Litellm)
-            .unwrap()
-            .into_json();
-        assert_eq!(direct_response, vertex_response);
-        assert_eq!(direct_response["model"], "mistral-ocr-maas");
-        assert_eq!(direct_response["object"], "ocr");
-        assert_eq!(direct_response["extra"], "preserved");
-    }
-
-    #[derive(serde::Deserialize)]
-    struct KnownParams {
-        pages: Option<Vec<i64>>,
-    }
-
-    #[rstest::rstest]
-    fn parsed_provider_params_separates_known_and_extra_params() {
-        let arguments: CallArguments = serde_json::from_value(json!({
-            "pages": [0, 2],
-            "future_ocr_option": true,
-            "extra_body": {"provider_option": "value"}
-        }))
-        .unwrap();
-        let known: KnownParams = parse_options(&arguments).unwrap();
-        assert_eq!(known.pages, Some(vec![0, 2]));
-        assert_eq!(arguments["future_ocr_option"], true);
-        assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
-        assert_eq!(
-            arguments
-                .iter()
-                .filter(|(name, _)| name.as_str() != "pages")
-                .count(),
-            2
-        );
-        assert_eq!(
-            compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
-            json!({
-                "pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
-            })
-        );
-    }
-}

litellm-rust/crates/inference-ocr/src/provider_config.rs

@@ -1,5 +1,6 @@
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
+use litellm_auth::{InputSource, SecretValue, Sourced};
+use litellm_core_utils::get_llm_provider_logic::{LlmProviders, get_custom_llm_provider};
+use litellm_inference::provider::{ResolvedProvider, resolve_llm_provider};
 use litellm_llms::{
     aws_textract::ocr::{
         analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation,
@@ -25,6 +26,7 @@ use litellm_llms::{
     },
 };
 use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat};
+use litellm_secrets::source::Secrets;
 
 macro_rules! with_config {
     ($kind:expr, $config:ident => $body:expr) => {
@@ -115,7 +117,18 @@ impl OcrConfigKind {
     }
 
     pub(crate) fn secret_names(self) -> Vec<&'static str> {
-        with_config!(self, config => config.secret_names())
+        let names = with_config!(self, config => config.secret_names());
+        let fallback_names: &[&str] = match self.provider() {
+            LlmProviders::Mistral => &["MISTRAL_AZURE_API_KEY", "MISTRAL_AZURE_API_BASE"],
+            LlmProviders::AzureAi => &["AZURE_AI_API_BASE"],
+            _ => &[],
+        };
+        let additional_names = fallback_names
+            .iter()
+            .copied()
+            .filter(|name| !names.contains(name))
+            .collect::<Vec<_>>();
+        names.into_iter().chain(additional_names).collect()
     }
 
     pub(crate) fn get_health_check_document(self) -> OcrDocument {
@@ -129,6 +142,43 @@ impl OcrConfigKind {
         with_config!(self, config => config.resolve_connection_params(inputs))
     }
 
+    pub(crate) fn resolve_credentials(
+        self,
+        credentials: OcrCredentialInputs,
+        secrets: &Secrets,
+    ) -> ResolvedOcrCredentials {
+        let (preferred_api_key_env, api_base_env) = match self.provider() {
+            LlmProviders::Mistral => (
+                Some("MISTRAL_AZURE_API_KEY"),
+                Some("MISTRAL_AZURE_API_BASE"),
+            ),
+            LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
+            _ => (None, None),
+        };
+        let secret = |name: &str| secrets.truthy(name);
+        let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
+            credentials.api_key.clone().or_else(|| {
+                preferred_api_key_env
+                    .into_iter()
+                    .chain(self.get_api_key_env_var())
+                    .find_map(secret)
+                    .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
+            })
+        });
+        let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
+            credentials.api_base.clone().or_else(|| {
+                api_base_env
+                    .and_then(secret)
+                    .map(|value| Sourced::new(value, InputSource::Environment))
+            })
+        });
+        self.resolve_connection_params(OcrCredentialInputs {
+            dynamic_api_key,
+            dynamic_api_base,
+            ..credentials
+        })
+    }
+
     pub(crate) async fn ocr(
         self,
         client: &OcrClient,
@@ -164,7 +214,7 @@ pub fn passthrough_response(
     endpoint: &str,
     body: &[u8],
 ) -> Result<Option<LiteLLMOcrResponse>, Error> {
-    let (model, config) = resolve_provider_config(model, Some("azure_ai"))?;
+    let (provider, config) = resolve_provider_config(model, Some("azure_ai"))?;
     let segments: Vec<&str> = endpoint
         .split('/')
         .filter(|segment| !segment.is_empty())
@@ -173,7 +223,7 @@ pub fn passthrough_response(
         OcrConfigKind::AzureAi => segments == AZURE_AI_OCR_PATH,
         OcrConfigKind::AzureCohere => segments == AZURE_COHERE_PARSE_PATH,
         OcrConfigKind::AzureDocumentIntelligence => {
-            segments == AzureDocumentIntelligenceOcrConfig::analyze_path(&model)?
+            segments == AzureDocumentIntelligenceOcrConfig::analyze_path(provider.model)?
         }
         other => {
             let provider: &'static str = other.provider().into();
@@ -183,24 +233,28 @@ pub fn passthrough_response(
     if !is_ocr_endpoint {
         return Ok(None);
     }
-    with_config!(config, config => config.transform_ocr_response(&model, body, OcrResponseFormat::Litellm))
+    with_config!(config, config => config.transform_ocr_response(provider.model, body, OcrResponseFormat::Litellm))
         .map(Some)
 }
 
-pub(crate) fn resolve_provider_config(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<(String, OcrConfigKind), Error> {
-    let provider =
-        get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider {
-            model,
-            custom_llm_provider: LlmProviders::Mistral.into(),
-        });
-    let llm_provider = provider
-        .custom_llm_provider
-        .parse::<LlmProviders>()
-        .map_err(|_| Error::InvalidProvider(provider.custom_llm_provider.to_string()))?;
-    let config = match llm_provider {
+pub(crate) fn resolve_provider_config<'a>(
+    model: &'a str,
+    custom_llm_provider: Option<&'a str>,
+) -> Result<(ResolvedProvider<'a>, OcrConfigKind), Error> {
+    let provider = match resolve_llm_provider(model, custom_llm_provider, "ocr") {
+        Ok(provider) => provider,
+        Err(_) if get_custom_llm_provider(model, custom_llm_provider).is_none() => {
+            ResolvedProvider {
+                model,
+                provider: LlmProviders::Mistral,
+            }
+        }
+        Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
+            return Err(Error::InvalidProvider(provider));
+        }
+        Err(error) => return Err(Error::InvalidProvider(error.to_string())),
+    };
+    let config = match provider.provider {
         LlmProviders::AwsTextract => match TextractOperation::from_model(provider.model)? {
             TextractOperation::DetectDocumentText => OcrConfigKind::AwsTextract,
             TextractOperation::AnalyzeDocument => OcrConfigKind::AwsTextractAnalyze,
@@ -227,11 +281,11 @@ pub(crate) fn resolve_provider_config(
         LlmProviders::VertexAi => OcrConfigKind::VertexAi,
         _ => {
             return Err(Error::InvalidProvider(
-                provider.custom_llm_provider.to_string(),
+                <&'static str>::from(provider.provider).to_string(),
             ));
         }
     };
-    Ok((provider.model.to_string(), config))
+    Ok((provider, config))
 }
 
 fn is_document_intelligence_model(model: &str) -> bool {
@@ -243,7 +297,7 @@ fn is_document_intelligence_model(model: &str) -> bool {
 mod tests {
     use std::collections::HashSet;
 
-    use litellm_auth::{InputSource, Sourced};
+    use litellm_auth::Sourced;
     use litellm_llms::{
         base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document,
     };
@@ -484,8 +538,8 @@ mod tests {
         #[case] expected_config: OcrConfigKind,
     ) {
         let expected_model = qualified_model.split_once('/').unwrap().1;
-        let (model, config) = resolve_provider_config(qualified_model, None).unwrap();
-        assert_eq!(model, expected_model);
+        let (provider, config) = resolve_provider_config(qualified_model, None).unwrap();
+        assert_eq!(provider.model, expected_model);
         assert_eq!(config, expected_config);
     }
 
@@ -528,7 +582,7 @@ mod tests {
             expected_config
         );
         assert_eq!(
-            resolve_provider_config(model, None).unwrap().0,
+            resolve_provider_config(model, None).unwrap().0.model,
             model.split_once('/').unwrap().1
         );
     }

litellm-rust/crates/inference-ocr/src/types.rs

@@ -75,6 +75,8 @@ impl OcrConnectionInputs {
 
 pub struct LiteLLMOcrRequest<D = OcrDocumentInput> {
     pub model: String,
+    pub(crate) requested_model: String,
+    pub(crate) custom_llm_provider: Option<String>,
     pub document: D,
     pub credentials: OcrCredentialInputs,
     pub transport: OcrTransportConfig,
@@ -91,7 +93,8 @@ impl LiteLLMOcrRequest {
         custom_llm_provider: Option<&str>,
         optional_params: CallArguments,
     ) -> Result<Self, Error> {
-        let (model, config) = resolve_provider_config(&model, custom_llm_provider)?;
+        let (provider, config) = resolve_provider_config(&model, custom_llm_provider)?;
+        let resolved_model = provider.model.to_owned();
         let default_transport = OcrTransportConfig::default();
         let max_response_bytes = optional_params
             .get("max_response_bytes")
@@ -116,7 +119,9 @@ impl LiteLLMOcrRequest {
             .collect();
 
         Ok(Self {
-            model,
+            model: resolved_model,
+            requested_model: model,
+            custom_llm_provider: custom_llm_provider.map(str::to_owned),
             document: document.into(),
             credentials: OcrCredentialInputs::default(),
             transport,
@@ -135,6 +140,8 @@ impl<D> LiteLLMOcrRequest<D> {
     ) -> Result<LiteLLMOcrRequest<T>, E> {
         Ok(LiteLLMOcrRequest {
             model: self.model,
+            requested_model: self.requested_model,
+            custom_llm_provider: self.custom_llm_provider,
             document: map(self.document)?,
             credentials: self.credentials,
             transport: self.transport,
@@ -148,6 +155,8 @@ impl<D> LiteLLMOcrRequest<D> {
     pub fn with_document<T>(self, document: T) -> LiteLLMOcrRequest<T> {
         LiteLLMOcrRequest {
             model: self.model,
+            requested_model: self.requested_model,
+            custom_llm_provider: self.custom_llm_provider,
             document,
             credentials: self.credentials,
             transport: self.transport,

litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs

@@ -66,6 +66,19 @@ async fn interception_runs_once_and_observers_receive_ordered_success_events() {
     assert_eq!(received(&upstream).await.len(), 1);
 }
 
+#[rstest::rstest]
+#[case::mutated_model("model-b")]
+#[tokio::test]
+async fn provider_request_uses_the_request_model_after_it_is_mutated(#[case] model: &str) {
+    let upstream = upstream([pages_response()]).await;
+    let mut request = ocr_request("mistral/model-a", &upstream.uri(), json!({}));
+    request.model = model.into();
+
+    perform(request).await.unwrap();
+
+    assert_eq!(only_request(&upstream).await.json()["model"], json!(model));
+}
+
 #[rstest::rstest]
 #[tokio::test]
 async fn a_blocking_before_send_prevents_the_call_and_emits_one_failure() {

litellm-rust/crates/inference-responses/AGENTS.md

@@ -2,3 +2,4 @@
 - This crate owns Responses API call orchestration. Payloads belong in `litellm-llms-types`, transformations in `llms/src/base_llm/responses` and `llms/src/<provider>/responses`
 - The HTTP route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks, cached through `litellm_inference::caching::execute_streaming`
 - WebSocket sessions (`src/websocket.rs`) stay separate from the HTTP call driver because a connection can accept multiple requests while receiving events
+- `src/ids.rs` owns the LiteLLM-managed ids handed back to callers (`resp_`, `encitem_`, `litellm_enc:`, `cntr_`) and must stay fully compatible with `litellm/responses/utils.py`

litellm-rust/crates/inference-responses/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+base64.workspace = true
 bytes.workspace = true
 futures-util.workspace = true
 litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }

litellm-rust/crates/inference-responses/src/handler.rs

@@ -3,39 +3,61 @@ use litellm_host::observation::ObservationSender;
 use std::time::Duration;
 
 use futures_util::StreamExt;
-use litellm_host::interceptors::{Interceptors, RawResponse, WireRequest};
-use litellm_llms::base_llm::auth::{Authenticated, resolve_auth};
+use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
+use litellm_llms::base_llm::auth::{AuthScheme, Authenticated, resolve_auth};
+use litellm_secrets::source::Secrets;
 
 use super::{
-    Error,
-    types::{ProviderResponsesRequest, ResponsesOutput, ResponsesStreamHead},
+    Error, ResponsesRoute,
+    types::{ResponsesCall, ResponsesOutput, ResponsesProvider, ResponsesStreamHead},
 };
 
 pub(super) async fn execute(
-    http: &litellm_http::Client,
-    auth: &litellm_auth::AuthServices,
-    request: ProviderResponsesRequest,
-    cache: Option<litellm_cache_response::ScopedCache>,
+    route: &ResponsesRoute,
+    ResponsesProvider { identity, config }: ResponsesProvider,
+    call: ResponsesCall,
+    secrets: Secrets,
     cache_options: Option<litellm_cache_response::CachePolicy>,
     interceptors: &impl Interceptors<Error>,
     observers: Option<&ObservationSender>,
 ) -> Result<ResponsesOutput, Error> {
-    let authenticated = resolve_auth(auth, request.environment, &|_| None).await?;
-    let identity = litellm_host::interceptors::ProviderIdentity {
-        model: request.context.model.clone(),
-        provider: request.context.custom_llm_provider.clone(),
+    let lookup = |name: &str| secrets.get(name);
+    let environment = config.validate_environment(
+        litellm_http::request::string_headers("responses", call.extra_headers)?,
+        call.api_key.as_deref(),
+        &lookup,
+    )?;
+    let url = config.get_complete_url(call.api_base.as_deref(), &lookup);
+    let context = RequestContext {
+        model: identity.model.clone(),
+        custom_llm_provider: identity.provider.clone(),
+        optional_params: serde_json::Value::Object(call.optional_params.clone()),
+        secret_fields: Vec::new(),
+        api_key: match &environment.auth {
+            AuthScheme::Credential { secret, .. } => Some(secret.clone()),
+            _ => None,
+        },
     };
+    let body = config.transform_responses_api_request(
+        &identity.model,
+        call.input,
+        call.optional_params,
+    )?;
+    let authenticated = resolve_auth(&route.auth, environment, &|_| None).await?;
     let wire = interceptors
         .before_provider_request(
             WireRequest {
-                url: request.url,
+                url,
                 headers: authenticated.headers,
-                body: request.body,
+                body,
             },
-            request.context,
+            context,
         )
         .await?;
-    let cache = cache.filter(|_| authenticated.signer.is_none());
+    let cache = route
+        .cache
+        .clone()
+        .filter(|_| authenticated.signer.is_none());
     let cache_request = litellm_inference::caching::CacheRequest::from_wire(
         identity,
         cache.as_ref().map(|_| &wire),
@@ -59,9 +81,9 @@ pub(super) async fn execute(
                 },
                 wire.url,
                 &wire.body,
-                Some(request.timeout.unwrap_or(Duration::from_secs(600))),
+                Some(call.timeout.unwrap_or(Duration::from_secs(600))),
             )?;
-            let response = litellm_inference::outbound::send(outbound, http)
+            let response = litellm_inference::outbound::send(outbound, &route.http)
                 .await
                 .map_err(network)?;
             let status = response.status().as_u16();
@@ -103,8 +125,7 @@ pub(super) async fn execute(
                 .map_err(Error::post_call)?;
             let value = serde_json::from_str(&body)
                 .map_err(|error| Error::InvalidResponse(error.to_string().into()))?;
-            request
-                .config
+            config
                 .transform_response_api_response(value)
                 .map(ResponsesOutput::Complete)
                 .map_err(Error::from)

litellm-rust/crates/inference-responses/src/ids.rs

@@ -0,0 +1,499 @@
+//! LiteLLM-managed identifiers on Responses payloads, fully compatible with `litellm/responses/utils.py`
+//!
+//! LiteLLM rewrites some ids in a Responses API result before handing it back, so that a later
+//! request carrying those ids can be routed to the provider and deployment that produced them,
+//! and then restored to what the provider originally issued. The native and Python paths serve
+//! the same traffic, so an id minted by either one must decode on the other.
+//!
+//! Each managed id is a fixed prefix followed by standard, padded base64 of a `;`-separated
+//! plaintext:
+//!
+//! | Id | Applied to | Decoded plaintext |
+//! |---|---|---|
+//! | `resp_<b64>` | the response `id` | `litellm:custom_llm_provider:{provider};model_id:{model_id};response_id:{id}` |
+//! | `encitem_<b64>` | the `id` of an output item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` |
+//! | `litellm_enc:<b64>;{content}` | that item's `encrypted_content` | `<b64>` is `model_id:{model_id}`, and the original content follows the `;` unencoded |
+//! | `cntr_<b64>` | `container_id` on output items, `code_interpreter_call`, and message annotations | `litellm:custom_llm_provider:{provider};model_id:{model_id};container_id:{id}` |
+//!
+//! For example, OpenAI's `resp_0a1b2c3d4e5f` served without a deployment id becomes
+//! `resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm`,
+//! which decodes to `litellm:custom_llm_provider:openai;model_id:None;response_id:resp_0a1b2c3d4e5f`.
+//!
+//! How they fit together:
+//! - `resp_` lets a follow-up's `previous_response_id` find the provider and deployment again.
+//!   [`encode_response`] stamps it on every response, unless the id is already managed.
+//! - `encitem_` and `litellm_enc:` carry encrypted-content affinity: encrypted reasoning only
+//!   decrypts under the API key that produced it, so the router pins replays to that deployment.
+//!   They are stamped only when the router sets `encrypted_content_affinity_enabled` and a
+//!   deployment `model_id` is known (see [`IdScope`]). Items without an `id`, such as Codex's,
+//!   carry the deployment in the wrapped content alone.
+//! - `cntr_` lets container calls find the provider and deployment that own a container. A
+//!   container id that already names a provider is left as it is.
+//! - On the way in, [`restore_input`] turns `encitem_` ids and `litellm_enc:` content in the
+//!   request input back into the provider's own bytes before the request is sent. The routing
+//!   decision itself stays in Python's `EncryptedContentAffinityCheck`.
+//!
+//! Python's quirks are kept on purpose:
+//! - A missing provider or `model_id` is written as `None` in `resp_` ids (an f-string of
+//!   `None`), but as an empty string in `cntr_` ids. [`decode_container_id`] reads both as absent.
+//! - Decoding follows Python's non-strict `base64.b64decode`: characters outside the alphabet
+//!   are skipped, trailing bits are allowed, and padding stripped in transit is added back.
+//! - A `resp_` id counts as managed when it decodes to at least three `;`-separated parts.
+
+use base64::{
+    Engine, alphabet,
+    engine::{GeneralPurpose, general_purpose::PAD},
+};
+use litellm_llms_types::formats::responses::ResponsesApiResponse;
+use serde_json::{Map, Value};
+
+const RESPONSE_PREFIX: &str = "resp_";
+const ITEM_PREFIX: &str = "encitem_";
+const CONTAINER_PREFIX: &str = "cntr_";
+const WRAPPED_CONTENT_PREFIX: &str = "litellm_enc:";
+
+/// Python's non-strict `base64.b64decode`: characters outside the alphabet are skipped and
+/// non-zero trailing bits are accepted
+const LENIENT: GeneralPurpose = GeneralPurpose::new(
+    &alphabet::STANDARD,
+    PAD.with_decode_allow_trailing_bits(true),
+);
+
+/// Where a response came from, as encoded into the ids LiteLLM hands back
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct IdScope {
+    pub provider: Option<String>,
+    pub model_id: Option<String>,
+    pub encrypted_content_affinity: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DecodedContainerId {
+    pub provider: Option<String>,
+    pub model_id: Option<String>,
+    pub container_id: String,
+}
+
+/// Rewrites the response id, encrypted-content items and container ids the way the Python
+/// Responses path does before returning to the caller. An already managed id is left untouched
+pub fn encode_response(response: &mut ResponsesApiResponse, scope: &IdScope) {
+    if is_managed_response_id(&response.id) {
+        return;
+    }
+    response.id = response_id(scope, &response.id);
+    if let Some(model_id) = scope
+        .model_id
+        .as_deref()
+        .filter(|_| scope.encrypted_content_affinity)
+        .filter(|model_id| !model_id.is_empty())
+    {
+        response
+            .output
+            .iter_mut()
+            .filter_map(Value::as_object_mut)
+            .for_each(|item| encode_encrypted_item(item, model_id));
+    }
+    response
+        .output
+        .iter_mut()
+        .for_each(|item| encode_containers(item, scope));
+}
+
+/// Restores encrypted-content item ids and contents that LiteLLM encoded on an earlier response,
+/// so the provider receives what it originally issued
+pub fn restore_input(input: &mut Value) {
+    let Some(items) = input.as_array_mut() else {
+        return;
+    };
+    for item in items.iter_mut().filter_map(Value::as_object_mut) {
+        if let Some(Value::String(id)) = item.get_mut("id")
+            && let Some((_, original)) = decode_encrypted_item_id(id)
+        {
+            *id = original;
+        }
+        if let Some(Value::String(content)) = item.get_mut("encrypted_content")
+            && let Some(original) = unwrap_encrypted_content(content)
+        {
+            *content = original;
+        }
+    }
+}
+
+pub fn response_id(scope: &IdScope, id: &str) -> String {
+    let assembled = format!(
+        "litellm:custom_llm_provider:{};model_id:{};response_id:{id}",
+        python_str(scope.provider.as_deref()),
+        python_str(scope.model_id.as_deref()),
+    );
+    format!("{RESPONSE_PREFIX}{}", LENIENT.encode(assembled))
+}
+
+pub fn is_managed_response_id(id: &str) -> bool {
+    decode_text(&id.replace(RESPONSE_PREFIX, ""))
+        .is_some_and(|decoded| decoded.split(';').count() >= 3)
+}
+
+pub fn encrypted_item_id(model_id: &str, id: &str) -> String {
+    format!(
+        "{ITEM_PREFIX}{}",
+        LENIENT.encode(format!("litellm:model_id:{model_id};item_id:{id}"))
+    )
+}
+
+pub fn decode_encrypted_item_id(id: &str) -> Option<(String, String)> {
+    let decoded = decode_text(&padded(id.strip_prefix(ITEM_PREFIX)?))?;
+    let (model_id, item_id) = decoded.split_once(';')?;
+    Some((
+        model_id.replace("litellm:model_id:", ""),
+        item_id.replace("item_id:", ""),
+    ))
+}
+
+pub fn wrap_encrypted_content(content: &str, model_id: &str) -> String {
+    format!(
+        "{WRAPPED_CONTENT_PREFIX}{};{content}",
+        LENIENT.encode(format!("model_id:{model_id}"))
+    )
+}
+
+/// The original content of a wrapped `encrypted_content`, or `None` when it is not wrapped
+pub fn unwrap_encrypted_content(content: &str) -> Option<String> {
+    if !content.starts_with(WRAPPED_CONTENT_PREFIX) {
+        return None;
+    }
+    let (metadata, original) = content.split_once(';')?;
+    decode_text(&padded(&metadata.replace(WRAPPED_CONTENT_PREFIX, "")))?;
+    Some(original.to_owned())
+}
+
+pub fn container_id(scope: &IdScope, id: &str) -> String {
+    let assembled = format!(
+        "litellm:custom_llm_provider:{};model_id:{};container_id:{id}",
+        scope.provider.as_deref().unwrap_or_default(),
+        scope.model_id.as_deref().unwrap_or_default(),
+    );
+    format!("{CONTAINER_PREFIX}{}", LENIENT.encode(assembled))
+}
+
+pub fn decode_container_id(id: &str) -> DecodedContainerId {
+    let unmanaged = || DecodedContainerId {
+        provider: None,
+        model_id: None,
+        container_id: id.to_owned(),
+    };
+    if !id.starts_with(CONTAINER_PREFIX) {
+        return unmanaged();
+    }
+    let Some(decoded) = decode_text(&id.replace(CONTAINER_PREFIX, "")) else {
+        return unmanaged();
+    };
+    let parsed = decoded
+        .strip_prefix("litellm:custom_llm_provider:")
+        .and_then(|rest| rest.split_once(';'))
+        .and_then(|(provider, rest)| {
+            let (model_id, rest) = rest.strip_prefix("model_id:")?.split_once(';')?;
+            let container_id = rest.strip_prefix("container_id:")?;
+            (!container_id.is_empty() && !container_id.contains('\n')).then_some((
+                provider,
+                model_id,
+                container_id,
+            ))
+        });
+    let Some((provider, model_id, container_id)) = parsed else {
+        return unmanaged();
+    };
+    let present = |value: &str| (!matches!(value, "" | "None")).then(|| value.to_owned());
+    DecodedContainerId {
+        provider: present(provider),
+        model_id: present(model_id),
+        container_id: container_id.to_owned(),
+    }
+}
+
+fn encode_encrypted_item(item: &mut Map<String, Value>, model_id: &str) {
+    let Some(Value::String(content)) = item.get_mut("encrypted_content") else {
+        return;
+    };
+    if content.is_empty() {
+        return;
+    }
+    *content = wrap_encrypted_content(content, model_id);
+    if let Some(Value::String(id)) = item.get_mut("id")
+        && !id.is_empty()
+    {
+        *id = encrypted_item_id(model_id, id);
+    }
+}
+
+fn encode_containers(item: &mut Value, scope: &IdScope) {
+    let Some(item) = item.as_object_mut() else {
+        return;
+    };
+    encode_container_field(item, scope);
+    if let Some(nested) = item
+        .get_mut("code_interpreter_call")
+        .and_then(Value::as_object_mut)
+    {
+        encode_container_field(nested, scope);
+    }
+    if item.get("type").and_then(Value::as_str) != Some("message") {
+        return;
+    }
+    let Some(content) = item.get_mut("content").and_then(Value::as_array_mut) else {
+        return;
+    };
+    content
+        .iter_mut()
+        .filter_map(|part| part.get_mut("annotations").and_then(Value::as_array_mut))
+        .flatten()
+        .for_each(|annotation| encode_containers(annotation, scope));
+}
+
+fn encode_container_field(item: &mut Map<String, Value>, scope: &IdScope) {
+    if let Some(Value::String(id)) = item.get_mut("container_id")
+        && decode_container_id(id).provider.is_none()
+    {
+        *id = container_id(scope, id);
+    }
+}
+
+fn python_str(value: Option<&str>) -> &str {
+    value.unwrap_or("None")
+}
+
+fn padded(value: &str) -> String {
+    let missing = value.len() % 4;
+    let padding = if missing == 0 { 0 } else { 4 - missing };
+    format!("{value}{}", "=".repeat(padding))
+}
+
+fn decode_text(value: &str) -> Option<String> {
+    let alphabet: String = value
+        .chars()
+        .filter(|character| {
+            character.is_ascii_alphanumeric() || matches!(character, '+' | '/' | '=')
+        })
+        .collect();
+    String::from_utf8(LENIENT.decode(alphabet).ok()?).ok()
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    fn scope(provider: Option<&str>, model_id: Option<&str>, affinity: bool) -> IdScope {
+        IdScope {
+            provider: provider.map(str::to_owned),
+            model_id: model_id.map(str::to_owned),
+            encrypted_content_affinity: affinity,
+        }
+    }
+
+    fn response(id: &str, output: Value) -> ResponsesApiResponse {
+        serde_json::from_value(json!({"id": id, "model": "gpt-6.1-sol", "output": output})).unwrap()
+    }
+
+    #[rstest]
+    #[case::openai_without_model_id(
+        Some("openai"),
+        None,
+        "resp_0a1b2c3d4e5f",
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm"
+    )]
+    #[case::openrouter_with_model_id(
+        Some("openrouter"),
+        Some("deploy-1"),
+        "gen-123",
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5yb3V0ZXI7bW9kZWxfaWQ6ZGVwbG95LTE7cmVzcG9uc2VfaWQ6Z2VuLTEyMw=="
+    )]
+    fn response_ids_match_python(
+        #[case] provider: Option<&str>,
+        #[case] model_id: Option<&str>,
+        #[case] id: &str,
+        #[case] expected: &str,
+    ) {
+        assert_eq!(response_id(&scope(provider, model_id, false), id), expected);
+    }
+
+    #[rstest]
+    #[case::raw_short_id("resp_0a1b2c3d4e5f", false)]
+    #[case::raw_long_id("resp_68a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3", false)]
+    #[case::two_segments("resp_YTti", false)]
+    #[case::three_segments("resp_YTtiO2M=", true)]
+    #[case::managed(
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOng=",
+        true
+    )]
+    fn managed_response_ids_match_python(#[case] id: &str, #[case] expected: bool) {
+        assert_eq!(is_managed_response_id(id), expected);
+    }
+
+    #[test]
+    fn encrypted_items_match_python() {
+        assert_eq!(
+            encrypted_item_id("deploy-1", "rs_0a1b"),
+            "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI="
+        );
+        assert_eq!(
+            wrap_encrypted_content("gAAAA", "deploy-1"),
+            "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA"
+        );
+    }
+
+    #[rstest]
+    #[case::padded("encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", Some(("deploy-1", "rs_0a1b")))]
+    #[case::padding_stripped_in_transit("encitem_bGl0ZWxsbTptb2RlbF9pZDptO2l0ZW1faWQ6cnNfMQ", Some(("m", "rs_1")))]
+    #[case::not_base64("encitem_notbase64!", None)]
+    #[case::unmanaged("rs_0a1b", None)]
+    fn encrypted_item_ids_decode_like_python(
+        #[case] id: &str,
+        #[case] expected: Option<(&str, &str)>,
+    ) {
+        assert_eq!(
+            decode_encrypted_item_id(id),
+            expected.map(|(model_id, item_id)| (model_id.to_owned(), item_id.to_owned()))
+        );
+    }
+
+    #[rstest]
+    #[case::unpadded_metadata("litellm_enc:bW9kZWxfaWQ6bQ;gAAAA", Some("gAAAA"))]
+    #[case::no_separator("litellm_enc:nosemicolon", None)]
+    #[case::unwrapped("gAAAA", None)]
+    fn encrypted_content_unwraps_like_python(
+        #[case] content: &str,
+        #[case] expected: Option<&str>,
+    ) {
+        assert_eq!(unwrap_encrypted_content(content).as_deref(), expected);
+    }
+
+    #[rstest]
+    #[case::with_model_id(
+        Some("deploy-1"),
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpkZXBsb3ktMTtjb250YWluZXJfaWQ6Y250cl9hYmM="
+    )]
+    #[case::without_model_id(
+        None,
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfYWJj"
+    )]
+    fn container_ids_match_python(#[case] model_id: Option<&str>, #[case] expected: &str) {
+        assert_eq!(
+            container_id(&scope(Some("openai"), model_id, false), "cntr_abc"),
+            expected
+        );
+    }
+
+    #[rstest]
+    #[case::raw("cntr_abc", None, None, "cntr_abc")]
+    #[case::managed(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDptO2NvbnRhaW5lcl9pZDpjbnRyXzE=",
+        Some("openai"),
+        Some("m"),
+        "cntr_1"
+    )]
+    #[case::empty_segments(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOjttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfMQ==",
+        None,
+        None,
+        "cntr_1"
+    )]
+    #[case::python_none_provider(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOk5vbmU7bW9kZWxfaWQ6O2NvbnRhaW5lcl9pZDp4",
+        None,
+        None,
+        "x"
+    )]
+    fn container_ids_decode_like_python(
+        #[case] id: &str,
+        #[case] provider: Option<&str>,
+        #[case] model_id: Option<&str>,
+        #[case] container_id: &str,
+    ) {
+        assert_eq!(
+            decode_container_id(id),
+            DecodedContainerId {
+                provider: provider.map(str::to_owned),
+                model_id: model_id.map(str::to_owned),
+                container_id: container_id.to_owned(),
+            }
+        );
+    }
+
+    #[test]
+    fn encodes_response_items_under_affinity() {
+        let mut encoded = response(
+            "resp_1",
+            json!([
+                {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
+                {"type": "reasoning", "encrypted_content": "gBBBB", "summary": []},
+                {"type": "code_interpreter_call", "id": "ci_1", "container_id": "cntr_abc"},
+                {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
+                    {"type": "container_file_citation", "container_id": "cntr_abc"}
+                ]}]}
+            ]),
+        );
+        let scope = scope(Some("openai"), Some("deploy-1"), true);
+        encode_response(&mut encoded, &scope);
+        let container = container_id(&scope, "cntr_abc");
+        assert_eq!(encoded.id, response_id(&scope, "resp_1"));
+        assert_eq!(
+            Value::Array(encoded.output),
+            json!([
+                {
+                    "type": "reasoning",
+                    "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
+                    "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
+                    "summary": []
+                },
+                {"type": "reasoning", "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gBBBB", "summary": []},
+                {"type": "code_interpreter_call", "id": "ci_1", "container_id": container},
+                {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
+                    {"type": "container_file_citation", "container_id": container}
+                ]}]}
+            ])
+        );
+    }
+
+    #[rstest]
+    #[case::affinity_disabled(scope(Some("openai"), Some("deploy-1"), false))]
+    #[case::no_model_id(scope(Some("openai"), None, true))]
+    fn leaves_encrypted_items_without_affinity(#[case] scope: IdScope) {
+        let output = json!([{"type": "reasoning", "id": "rs_1", "encrypted_content": "gAAAA", "summary": []}]);
+        let mut encoded = response("resp_1", output.clone());
+        encode_response(&mut encoded, &scope);
+        assert_eq!(Value::Array(encoded.output), output);
+    }
+
+    #[test]
+    fn leaves_managed_responses_untouched() {
+        let scope = scope(Some("openai"), Some("deploy-1"), true);
+        let id = response_id(&scope, "resp_1");
+        let output = json!([{"type": "code_interpreter_call", "container_id": "cntr_abc"}]);
+        let mut encoded = response(&id, output.clone());
+        encode_response(&mut encoded, &scope);
+        assert_eq!((encoded.id, Value::Array(encoded.output)), (id, output));
+    }
+
+    #[test]
+    fn restores_encrypted_items_in_input() {
+        let mut input = json!([
+            {
+                "type": "reasoning",
+                "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
+                "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
+                "summary": []
+            },
+            {"role": "user", "content": "hi"}
+        ]);
+        restore_input(&mut input);
+        assert_eq!(
+            input,
+            json!([
+                {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
+                {"role": "user", "content": "hi"}
+            ])
+        );
+    }
+}

litellm-rust/crates/inference-responses/src/lib.rs

@@ -1,19 +1,36 @@
 use litellm_host::observation::ObservationSender;
 pub use litellm_inference::RouteError as Error;
 
+pub mod ids;
 pub mod route;
 pub mod types;
 pub mod websocket;
 
 mod handler;
-mod prepare;
 
 use std::sync::Arc;
 
 use litellm_auth::AuthServices;
+use litellm_core_utils::get_llm_provider_logic::{
+    CustomLlmProvider, LlmProviders, get_custom_llm_provider,
+};
 use litellm_host::interceptors::Interceptors;
+use litellm_host::interceptors::ProviderIdentity;
+use litellm_llms::{
+    base_llm::responses::transformation::BaseResponsesApiConfig,
+    openai::responses::transformation::OpenAiResponsesApiConfig,
+    openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
+};
 use litellm_secrets::source::SecretSource;
-use types::{ResponsesCall, ResponsesOutput};
+use types::{ResponsesCall, ResponsesOutput, ResponsesProvider};
+
+fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> {
+    match provider {
+        LlmProviders::Openai => Some(&OpenAiResponsesApiConfig),
+        LlmProviders::Openrouter => Some(&OPENROUTER_RESPONSES_CONFIG),
+        _ => None,
+    }
+}
 
 #[derive(Clone)]
 pub struct ResponsesRoute {
@@ -77,34 +94,68 @@ impl ResponsesRoute {
         observers: Option<&ObservationSender>,
     ) -> Result<ResponsesOutput, Error> {
         litellm_inference::diagnostic::call(async {
-            self.run_provider(call, cache_options, interceptors, observers)
-                .await
+            let explicit = call
+                .custom_llm_provider
+                .as_deref()
+                .filter(|provider| !provider.is_empty());
+            let (provider_name, resolved_model) =
+                match get_custom_llm_provider(&call.model, explicit) {
+                    Some(CustomLlmProvider {
+                        model,
+                        custom_llm_provider,
+                    }) => (custom_llm_provider, model),
+                    None => ("openai", call.model.as_str()),
+                };
+            if resolved_model.is_empty() {
+                return Err(Error::InvalidProvider(call.model));
+            }
+            let provider = provider_name
+                .parse::<LlmProviders>()
+                .map_err(|_| Error::Unsupported("native HTTP responses provider"))?;
+            let config = responses_provider(provider)
+                .ok_or(Error::Unsupported("native HTTP responses provider"))?;
+            let wire_model = match provider {
+                LlmProviders::Openai if resolved_model.contains('/') => {
+                    return Err(Error::InvalidProvider(call.model));
+                }
+                LlmProviders::Openai => resolved_model.into(),
+                LlmProviders::Openrouter if explicit.is_some() && !resolved_model.contains('/') => {
+                    format!("openrouter/{resolved_model}")
+                }
+                LlmProviders::Openrouter => resolved_model.into(),
+                _ => return Err(Error::Unsupported("native HTTP responses provider")),
+            };
+            let provider = ResponsesProvider {
+                identity: ProviderIdentity {
+                    model: wire_model,
+                    provider: provider_name.into(),
+                },
+                config,
+            };
+            litellm_inference::diagnostic::provider(
+                &provider.identity.model,
+                &provider.identity.provider,
+            );
+            let secrets = self
+                .secrets
+                .resolve(
+                    provider
+                        .config
+                        .secret_names(call.api_key.as_deref(), call.api_base.as_deref()),
+                )
+                .await?;
+            let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
+                Box::pin(handler::execute(
+                    self,
+                    provider,
+                    call,
+                    secrets,
+                    cache_options,
+                    interceptors,
+                    observers,
+                ));
+            execute.await
         })
         .await
     }
-
-    async fn run_provider(
-        &self,
-        call: ResponsesCall,
-        cache_options: Option<litellm_cache_response::CachePolicy>,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<ResponsesOutput, Error> {
-        let request = prepare::prepare(call, self.secrets.as_ref()).await?;
-        litellm_inference::diagnostic::provider(
-            &request.context.model,
-            &request.context.custom_llm_provider,
-        );
-        let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
-            Box::pin(handler::execute(
-                &self.http,
-                &self.auth,
-                request,
-                self.cache.clone(),
-                cache_options,
-                interceptors,
-                observers,
-            ));
-        execute.await
-    }
 }

litellm-rust/crates/inference-responses/src/prepare.rs

@@ -1,162 +0,0 @@
-use litellm_core_utils::get_llm_provider_logic::{
-    CustomLlmProvider, LlmProviders, get_custom_llm_provider,
-};
-use litellm_host::interceptors::{ProviderIdentity, RequestContext};
-use litellm_llms::{
-    base_llm::responses::transformation::BaseResponsesApiConfig,
-    openai::responses::transformation::OpenAiResponsesApiConfig,
-    openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
-};
-use litellm_secrets::source::SecretSource;
-use serde_json::Value;
-
-use super::{
-    Error,
-    types::{ProviderResponsesRequest, ResponsesCall},
-};
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) async fn prepare(
-    call: ResponsesCall,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderResponsesRequest, Error> {
-    let ResponsesProvider { identity, config } =
-        resolve_provider(&call.model, call.custom_llm_provider.as_deref())?;
-    let provider = identity.provider.as_str();
-    let model = identity.model.as_str();
-    let snapshot = secrets
-        .resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref()))
-        .await?;
-    let lookup = |name: &str| snapshot.get(name);
-    let environment = config.validate_environment(
-        litellm_http::request::string_headers("responses", call.extra_headers)?,
-        call.api_key.as_deref(),
-        &lookup,
-    )?;
-    let context = RequestContext {
-        model: model.into(),
-        custom_llm_provider: provider.into(),
-        optional_params: Value::Object(call.optional_params.clone()),
-        secret_fields: Vec::new(),
-        api_key: match &environment.auth {
-            litellm_llms::base_llm::auth::AuthScheme::Credential { secret, .. } => {
-                Some(secret.clone())
-            }
-            _ => None,
-        },
-    };
-    let body = config.transform_responses_api_request(model, call.input, call.optional_params)?;
-    Ok(ProviderResponsesRequest {
-        url: config.get_complete_url(call.api_base.as_deref(), &lookup),
-        config,
-        environment,
-        body,
-        context,
-        timeout: call.timeout,
-    })
-}
-
-pub(super) struct ResponsesProvider {
-    pub identity: ProviderIdentity,
-    pub config: &'static dyn BaseResponsesApiConfig,
-}
-
-/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter
-/// provider a model without a vendor segment keeps the `openrouter/` prefix as its wire id.
-pub(super) fn resolve_provider(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<ResponsesProvider, Error> {
-    let explicit = custom_llm_provider.filter(|provider| !provider.is_empty());
-    let (provider, resolved) = match get_custom_llm_provider(model, explicit) {
-        Some(CustomLlmProvider {
-            model,
-            custom_llm_provider,
-        }) => (custom_llm_provider, model),
-        None => ("openai", model),
-    };
-    if resolved.is_empty() {
-        return Err(Error::InvalidProvider(model.into()));
-    }
-    let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) =
-        match provider.parse::<LlmProviders>() {
-            Ok(LlmProviders::Openai) if resolved.contains('/') => {
-                return Err(Error::InvalidProvider(model.into()));
-            }
-            Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()),
-            Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => (
-                &OPENROUTER_RESPONSES_CONFIG,
-                format!("openrouter/{resolved}"),
-            ),
-            Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()),
-            _ => return Err(Error::Unsupported("native HTTP responses provider")),
-        };
-    Ok(ResponsesProvider {
-        identity: ProviderIdentity {
-            model: wire_model,
-            provider: provider.into(),
-        },
-        config,
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use rstest::rstest;
-
-    use super::*;
-
-    /// Pinned against Python's `get_llm_provider` on 2026-10-11.
-    #[rstest]
-    #[case::bare("gpt-5", None, "openai", "gpt-5")]
-    #[case::openai_prefix("openai/gpt-5", None, "openai", "gpt-5")]
-    #[case::openai_explicit("gpt-5", Some("openai"), "openai", "gpt-5")]
-    #[case::openrouter_prefix(
-        "openrouter/anthropic/claude",
-        None,
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_explicit(
-        "openrouter/anthropic/claude",
-        Some("openrouter"),
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_explicit_unprefixed(
-        "anthropic/claude",
-        Some("openrouter"),
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_native_id(
-        "openrouter/auto",
-        Some("openrouter"),
-        "openrouter",
-        "openrouter/auto"
-    )]
-    #[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")]
-    fn providers_and_wire_models_match_python(
-        #[case] model: &str,
-        #[case] custom_llm_provider: Option<&str>,
-        #[case] provider: &str,
-        #[case] wire_model: &str,
-    ) {
-        let resolved = resolve_provider(model, custom_llm_provider).unwrap();
-        assert_eq!(resolved.identity.provider, provider);
-        assert_eq!(resolved.identity.model, wire_model);
-    }
-
-    #[rstest]
-    #[case::other_prefix("other/model", None)]
-    #[case::other_explicit("model", Some("other"))]
-    #[case::openai_with_vendor_segment("openai/other/model", None)]
-    #[case::empty_model("openai/", None)]
-    #[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
-    fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) {
-        assert!(matches!(
-            resolve_provider(model, custom_llm_provider),
-            Err(Error::InvalidProvider(_) | Error::Unsupported(_))
-        ));
-    }
-}

litellm-rust/crates/inference-responses/src/types.rs

@@ -2,9 +2,8 @@ use std::time::Duration;
 
 use bytes::Bytes;
 use litellm_host::call::CallOutput;
-use litellm_llms::base_llm::{
-    auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig,
-};
+use litellm_host::interceptors::ProviderIdentity;
+use litellm_llms::base_llm::responses::transformation::BaseResponsesApiConfig;
 use litellm_llms_types::formats::responses::ResponsesApiResponse;
 use serde_json::{Map, Value};
 
@@ -21,17 +20,13 @@ pub struct ResponsesCall {
     pub timeout: Option<Duration>,
 }
 
+pub(crate) struct ResponsesProvider {
+    pub identity: ProviderIdentity,
+    pub config: &'static dyn BaseResponsesApiConfig,
+}
+
 pub struct ResponsesStreamHead {
     pub headers: Vec<(String, String)>,
 }
 
 pub type ResponsesOutput = CallOutput<ResponsesApiResponse, ResponsesStreamHead, Bytes, Error>;
-
-pub(super) struct ProviderResponsesRequest {
-    pub config: &'static dyn BaseResponsesApiConfig,
-    pub environment: ValidatedEnvironment,
-    pub url: String,
-    pub body: Value,
-    pub context: litellm_host::interceptors::RequestContext,
-    pub timeout: Option<Duration>,
-}

litellm-rust/crates/inference-responses/tests/responses.rs

@@ -7,6 +7,7 @@ use std::sync::Arc;
 use futures_util::TryStreamExt;
 use litellm_host::{call::HostedCompletion, lifecycle::CallEvent};
 use litellm_inference_responses::{
+    Error,
     route::Responses,
     types::{ResponsesCall, ResponsesOutput},
 };
@@ -32,6 +33,110 @@ fn call() -> ResponsesCall {
     }
 }
 
+#[rstest]
+#[case::bare_openai("gpt-5", None, "gpt-5")]
+#[case::openai_prefix("openai/gpt-5", None, "gpt-5")]
+#[case::empty_explicit_provider("openai/gpt-5", Some(""), "gpt-5")]
+#[case::explicit_openai("gpt-5", Some("openai"), "gpt-5")]
+#[case::openrouter_prefix("openrouter/anthropic/claude", None, "anthropic/claude")]
+#[case::explicit_openrouter_prefix(
+    "openrouter/anthropic/claude",
+    Some("openrouter"),
+    "anthropic/claude"
+)]
+#[case::explicit_openrouter_unprefixed("anthropic/claude", Some("openrouter"), "anthropic/claude")]
+#[case::explicit_openrouter_native_id("auto", Some("openrouter"), "openrouter/auto")]
+#[case::implicit_openrouter_native_id("openrouter/auto", None, "auto")]
+#[tokio::test]
+async fn provider_resolution_preserves_the_wire_model(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+    #[case] expected_wire_model: &str,
+) {
+    let response = json!({
+        "id": "response-1",
+        "model": expected_wire_model,
+        "output": [{"type": "message", "content": []}],
+        "usage": {"total_tokens": 7}
+    });
+    let upstream = upstream([json_response(response)]).await;
+    responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_key: Some("test-key".into()),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(
+        only_request(&upstream).await.json()["model"],
+        expected_wire_model
+    );
+}
+
+#[rstest]
+#[case::explicit_unknown_provider("test-model", Some("unknown-provider"))]
+#[case::unsupported_responses_provider("test-model", Some("anthropic"))]
+#[tokio::test]
+async fn unsupported_providers_fail_before_transport(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+) {
+    let upstream = upstream([]).await;
+    let result = responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await;
+
+    assert!(matches!(
+        result,
+        Err(Error::Unsupported("native HTTP responses provider"))
+    ));
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[case::openai_with_vendor_segment("openai/other/model", None)]
+#[case::empty_model("openai/", None)]
+#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
+#[tokio::test]
+async fn invalid_provider_models_fail_before_transport(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+) {
+    let upstream = upstream([]).await;
+    let result = responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await;
+
+    assert!(matches!(result, Err(Error::InvalidProvider(_))));
+    assert!(received(&upstream).await.is_empty());
+}
+
 #[rstest]
 #[case::direct(false)]
 #[case::hosted(true)]

litellm-rust/crates/inference-transcription/src/handler.rs

@@ -1,28 +1,57 @@
 use std::time::Duration;
 
-use litellm_http::{Client, request::truncate_error_body};
-use litellm_llms::base_llm::auth::resolve_auth;
+use litellm_http::{
+    Client,
+    request::{string_headers, truncate_error_body, with_default_headers},
+};
+use litellm_llms::base_llm::{
+    audio_transcription::transformation::BaseAudioTranscriptionConfig,
+    auth::{ValidatedEnvironment, resolve_auth},
+};
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
 use super::Error;
-use crate::{
-    constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::ProviderAudioTranscriptionRequest,
-};
+use crate::{constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::AudioTranscriptionRequest};
+use litellm_inference::provider::ResolvedProvider;
 
-pub async fn execute_audio_transcription_provider_call(
+pub(super) async fn execute(
     http: &Client,
     auth: &litellm_auth::AuthServices,
-    request: ProviderAudioTranscriptionRequest,
+    config: &'static dyn BaseAudioTranscriptionConfig,
+    provider: ResolvedProvider<'_>,
+    call: AudioTranscriptionRequest<'_>,
+    secrets: Secrets,
 ) -> Result<Value, Error> {
-    let env_lookup = |key: &str| request.secrets.get(key);
-    let authenticated = resolve_auth(auth, request.environment.clone(), &env_lookup).await?;
+    let env_lookup = |key: &str| secrets.get(key);
+    let forwarded = string_headers("audio transcription", call.extra_headers)?;
+    let validated = config.validate_environment(
+        forwarded,
+        provider.model,
+        &call.optional_params,
+        &env_lookup,
+    )?;
+    let environment = ValidatedEnvironment {
+        headers: with_default_headers(validated.headers, config.default_headers()),
+        auth: validated.auth,
+    };
+    let url = config.get_complete_url(
+        call.api_base,
+        provider.model,
+        &call.optional_params,
+        &env_lookup,
+    )?;
+    let filtered_params = config.map_transcription_params(&call.optional_params);
+    let body = config
+        .transform_audio_transcription_request(provider.model, call.audio, filtered_params)?
+        .body;
+    let authenticated = resolve_auth(auth, environment, &env_lookup).await?;
     let outbound = litellm_inference::outbound::outbound_request(
         authenticated,
-        request.url.clone(),
-        &request.body,
+        url,
+        &body,
         Some(
-            request
-                .timeout
+            call.timeout
                 .unwrap_or(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)),
         ),
     )?;
@@ -47,8 +76,7 @@ pub async fn execute_audio_transcription_provider_call(
             error,
         ))
     })?;
-    Ok(request
-        .config
-        .transform_audio_transcription_response(&request.model, response_json)?
+    Ok(config
+        .transform_audio_transcription_response(provider.model, response_json)?
         .into_json())
 }

litellm-rust/crates/inference-transcription/src/lib.rs

@@ -2,16 +2,29 @@ pub mod types;
 pub use litellm_inference::RouteError as Error;
 mod constants;
 mod handler;
-mod prepare;
-pub use handler::execute_audio_transcription_provider_call;
+
 use litellm_auth::AuthServices;
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use litellm_inference::provider::resolve_llm_provider;
+use litellm_llms::{
+    base_llm::audio_transcription::transformation::BaseAudioTranscriptionConfig,
+    bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
+};
 use litellm_secrets::source::SecretSource;
-pub use prepare::prepare_audio_transcription_provider_call;
 use serde_json::Value;
 use std::sync::Arc;
 
 use crate::types::AudioTranscriptionRequest;
 
+fn audio_transcription_provider(
+    provider: LlmProviders,
+) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
+    match provider {
+        LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
+        _ => None,
+    }
+}
+
 #[derive(Clone)]
 pub struct AudioTranscriptionRoute {
     http: litellm_http::Client,
@@ -32,6 +45,10 @@ impl AudioTranscriptionRoute {
         }
     }
 
+    pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
+        self.run(request).await
+    }
+
     #[tracing::instrument(name = "litellm.route", skip_all, fields(
         route = "audio_transcription",
         model = request.model,
@@ -40,13 +57,23 @@ impl AudioTranscriptionRoute {
         stream = false,
         outcome
     ))]
-    pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
+    async fn run(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
         litellm_inference::diagnostic::unary(async {
-            let request =
-                prepare_audio_transcription_provider_call(request, self.secrets.as_ref()).await?;
-            litellm_inference::diagnostic::provider(&request.model, &request.custom_llm_provider);
+            let provider = resolve_llm_provider(
+                request.model,
+                request.custom_llm_provider,
+                "audio transcription",
+            )?;
+            let config = audio_transcription_provider(provider.provider).ok_or_else(|| {
+                Error::InvalidProvider(<&str>::from(provider.provider).to_string())
+            })?;
+            litellm_inference::diagnostic::provider(
+                provider.model,
+                <&'static str>::from(provider.provider),
+            );
+            let secrets = self.secrets.resolve(&config.secret_names()).await?;
             let execute: futures_util::future::BoxFuture<'_, Result<Value, Error>> = Box::pin(
-                execute_audio_transcription_provider_call(&self.http, &self.auth, request),
+                handler::execute(&self.http, &self.auth, config, provider, request, secrets),
             );
             execute.await
         })

litellm-rust/crates/inference-transcription/src/prepare.rs

@@ -1,65 +0,0 @@
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_http::request::string_headers;
-use litellm_http::request::with_default_headers;
-use litellm_llms::{
-    base_llm::{
-        audio_transcription::transformation::BaseAudioTranscriptionConfig,
-        auth::ValidatedEnvironment,
-    },
-    bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
-};
-use litellm_secrets::source::SecretSource;
-
-use super::Error;
-use crate::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest};
-use litellm_inference::provider::resolve_llm_provider;
-
-fn provider_config(provider: LlmProviders) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
-    match provider {
-        LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
-        _ => None,
-    }
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub async fn prepare_audio_transcription_provider_call(
-    request: AudioTranscriptionRequest<'_>,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderAudioTranscriptionRequest, Error> {
-    let provider_info = resolve_llm_provider(
-        request.model,
-        request.custom_llm_provider,
-        "audio transcription",
-    )?;
-    let model = provider_info.model.to_string();
-    let config = provider_config(provider_info.provider)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?;
-    let snapshot = secrets.resolve(&config.secret_names()).await?;
-    let env_lookup = |key: &str| snapshot.get(key);
-    let forwarded = string_headers("audio transcription", request.extra_headers)?;
-    let validated =
-        config.validate_environment(forwarded, &model, &request.optional_params, &env_lookup)?;
-    let environment = ValidatedEnvironment {
-        headers: with_default_headers(validated.headers, config.default_headers()),
-        auth: validated.auth,
-    };
-    let url = config.get_complete_url(
-        request.api_base,
-        &model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    let filtered_params = config.map_transcription_params(&request.optional_params);
-    let transformed =
-        config.transform_audio_transcription_request(&model, request.audio, filtered_params)?;
-    Ok(ProviderAudioTranscriptionRequest {
-        model,
-        custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
-        config,
-        url,
-        body: transformed.body,
-        environment,
-        secrets: snapshot,
-        timeout: request.timeout,
-    })
-}

litellm-rust/crates/inference-transcription/src/types.rs

@@ -1,9 +1,5 @@
-use litellm_secrets::source::Secrets;
 use std::time::Duration;
 
-use litellm_llms::base_llm::{
-    audio_transcription::transformation::BaseAudioTranscriptionConfig, auth::ValidatedEnvironment,
-};
 use serde_json::{Map, Value};
 
 pub struct AudioTranscriptionRequest<'a> {
@@ -16,37 +12,3 @@ pub struct AudioTranscriptionRequest<'a> {
     pub optional_params: Map<String, Value>,
     pub timeout: Option<Duration>,
 }
-
-#[derive(Clone)]
-pub struct ProviderAudioTranscriptionRequest {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseAudioTranscriptionConfig,
-    pub url: String,
-    pub body: Value,
-    pub environment: ValidatedEnvironment,
-    pub secrets: Secrets,
-    pub timeout: Option<Duration>,
-}
-
-impl ProviderAudioTranscriptionRequest {
-    pub fn model(&self) -> &str {
-        &self.model
-    }
-
-    pub fn custom_llm_provider(&self) -> &str {
-        &self.custom_llm_provider
-    }
-
-    pub fn url(&self) -> &str {
-        &self.url
-    }
-
-    pub fn body(&self) -> &Value {
-        &self.body
-    }
-
-    pub fn with_body(self, body: Value) -> Self {
-        Self { body, ..self }
-    }
-}

litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs

@@ -195,9 +195,10 @@ impl AnthropicBeta {
 
     fn rejected_by(&self) -> &'static [BetaProvider] {
         match self {
-            Self::AdvancedToolUse20251120 | Self::ContextManagement20250627 => {
-                &[BetaProvider::BedrockConverse]
+            Self::AdvancedToolUse20251120 => {
+                &[BetaProvider::BedrockConverse, BetaProvider::Databricks]
             }
+            Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
             Self::AdvisorTool20260301 | Self::Compact20260904 => &[
                 BetaProvider::AzureAi,
                 BetaProvider::BedrockConverse,
@@ -231,12 +232,11 @@ impl AnthropicBeta {
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::VertexAi,
+                BetaProvider::Databricks,
             ],
             Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
             Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
-            Self::DangerousToolUse20260903 => {
-                &[BetaProvider::BedrockConverse, BetaProvider::Databricks]
-            }
+            Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
             Self::Effort20251124 => &[BetaProvider::VertexAi],
             Self::FastMode20260201 | Self::Oauth20250420 => &[
                 BetaProvider::AzureAi,
@@ -244,6 +244,7 @@ impl AnthropicBeta {
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::VertexAi,
+                BetaProvider::Databricks,
             ],
             Self::FineGrainedToolStreaming20250514 => {
                 &[BetaProvider::AzureAi, BetaProvider::VertexAi]

litellm-rust/crates/llms/src/anthropic/beta_headers.rs

@@ -76,7 +76,7 @@ mod tests {
     #[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))]
     #[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))]
     #[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))]
-    #[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))]
+    #[case::databricks(BetaProvider::Databricks, None)]
     #[case::bedrock_converse(BetaProvider::BedrockConverse, None)]
     fn filter_renames_per_host(
         #[case] provider: BetaProvider,

litellm-rust/crates/llms/src/anthropic/messages/transformation.rs

@@ -74,6 +74,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -109,7 +110,10 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
     }
 
     fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
-        update_headers_with_anthropic_beta(headers, request)
+        merge_beta_headers(
+            headers,
+            provider_feature_betas(request, BetaProvider::Anthropic),
+        )
     }
 }
 
@@ -136,12 +140,20 @@ pub(crate) fn transform_messages_request(
         messages: strip_encrypted_reasoning_blocks(messages),
         params: MessagesOptionalParams {
             context_management,
+            extra: without_null_safeguards(request.params.extra),
             ..request.params
         },
         ..request
     })
 }
 
+fn without_null_safeguards(extra: Map<String, Value>) -> Map<String, Value> {
+    extra
+        .into_iter()
+        .filter(|(key, value)| key != "safeguards" || !value.is_null())
+        .collect()
+}
+
 pub(crate) fn update_headers_with_anthropic_beta(
     headers: Headers,
     request: &MessagesRequest,
@@ -149,12 +161,34 @@ pub(crate) fn update_headers_with_anthropic_beta(
     merge_beta_headers(headers, feature_betas(request))
 }
 
-/// The betas a request needs on a host other than the first-party API: the Anthropic set
-/// filtered and renamed through the host's `BetaProvider` policy.
+pub(crate) fn update_headers_with_safeguards_beta(
+    headers: Headers,
+    request: &MessagesRequest,
+    provider: BetaProvider,
+) -> Headers {
+    merge_beta_headers(
+        headers,
+        feature_betas(request).union(on_provider(mapped_host_betas(&request.params), provider)),
+    )
+}
+
+/// The betas a request needs on a host with a column in the beta map, filtered and renamed
+/// through that column.
 pub(crate) fn provider_feature_betas(request: &MessagesRequest, provider: BetaProvider) -> BetaSet {
-    feature_betas(request)
-        .iter()
-        .filter_map(|beta| beta.on(provider))
+    on_provider(
+        feature_betas(request).union(mapped_host_betas(&request.params)),
+        provider,
+    )
+}
+
+fn on_provider(betas: BetaSet, provider: BetaProvider) -> BetaSet {
+    betas.iter().filter_map(|beta| beta.on(provider)).collect()
+}
+
+fn mapped_host_betas(params: &MessagesOptionalParams) -> BetaSet {
+    carries_safeguards(params)
+        .then_some(AnthropicBeta::DangerousToolUse20260903)
+        .into_iter()
         .collect()
 }
 
@@ -206,6 +240,13 @@ fn uses_structured_output(params: &MessagesOptionalParams) -> bool {
             .is_some_and(|config| config.format.is_some())
 }
 
+fn carries_safeguards(params: &MessagesOptionalParams) -> bool {
+    params
+        .extra
+        .get("safeguards")
+        .is_some_and(|value| !value.is_null())
+}
+
 fn messages_carry_output_config(messages: &[Message]) -> bool {
     messages
         .iter()
@@ -847,6 +888,7 @@ mod tests {
             headers(forwarded),
             api_key,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &env(vars),
         )
@@ -1025,10 +1067,44 @@ mod tests {
         json!({"messages": [{"role": "user", "content": "hi", "output_config": null}]}),
         &["per-turn-control-2026-07-01"]
     )]
+    #[case::safeguards(
+        json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
+        &[]
+    )]
     fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
         assert_eq!(feature_betas(&request(fields)), betas(expected));
     }
 
+    #[rstest]
+    #[case::anthropic(BetaProvider::Anthropic, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
+    #[case::vertex_ai(BetaProvider::VertexAi, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
+    #[case::empty_list(BetaProvider::Anthropic, json!([]), &["dangerous-tool-use-2026-09-03"])]
+    #[case::not_a_list(BetaProvider::Anthropic, json!(1), &["dangerous-tool-use-2026-09-03"])]
+    #[case::null(BetaProvider::Anthropic, Value::Null, &[])]
+    #[case::bedrock_converse_has_a_null_cell(BetaProvider::BedrockConverse, json!([{"type": "dangerous_tool_use"}]), &[])]
+    fn safeguards_add_the_dangerous_tool_use_beta_where_the_beta_map_has_a_cell(
+        #[case] provider: BetaProvider,
+        #[case] safeguards: Value,
+        #[case] expected: &[&str],
+    ) {
+        assert_eq!(
+            provider_feature_betas(&request(json!({"safeguards": safeguards})), provider),
+            betas(expected)
+        );
+    }
+
+    #[rstest]
+    fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
+        let client = headers(&[("anthropic-beta", "claude-code-20250219")]);
+        assert_eq!(
+            update_headers_with_anthropic_beta(
+                client.clone(),
+                &request(json!({"safeguards": [{"type": "dangerous_tool_use"}]}))
+            ),
+            client
+        );
+    }
+
     #[rstest]
     #[case::no_betas(&[("x-api-key", "k"), ("anthropic-version", "2023-06-01")], json!({}), &[("x-api-key", "k"), ("anthropic-version", "2023-06-01")])]
     #[case::blank_beta_header(&[("Anthropic-Beta", " , "), ("x-api-key", "k")], json!({}), &[("Anthropic-Beta", " , "), ("x-api-key", "k")])]
@@ -1067,6 +1143,11 @@ mod tests {
         json!({"messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}]}),
         &[("anthropic-beta", "claude-code-20250219,context-management-2025-06-27,effort-2025-11-24,interleaved-thinking-2025-05-14,per-turn-control-2026-07-01")],
     )]
+    #[case::safeguards_add_the_dangerous_tool_use_beta_to_the_client_betas(
+        &[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")],
+        json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14")],
+    )]
     fn request_headers_merge_the_feature_betas(
         #[case] input: &[(&str, &str)],
         #[case] fields: Value,
@@ -1127,6 +1208,7 @@ mod tests {
             Vec::new(),
             None,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &record,
         );

litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs

@@ -1,8 +1,8 @@
 use litellm_auth::SecretValue;
 use litellm_http::request::{has_bearer_auth, has_header};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
-    providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
+    providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
 };
 use litellm_router_types::LitellmParams;
 
@@ -12,7 +12,7 @@ use crate::{
         common_utils::API_KEY_PLACEMENT,
         messages::{
             handler::shape_anthropic_messages_request,
-            transformation::{transform_messages_request, update_headers_with_anthropic_beta},
+            transformation::{transform_messages_request, update_headers_with_safeguards_beta},
         },
     },
     azure_ai::common_utils::{
@@ -83,6 +83,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -104,7 +105,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
     }
 
     fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
-        update_headers_with_anthropic_beta(headers, request)
+        update_headers_with_safeguards_beta(headers, request, BetaProvider::AzureAi)
     }
 }
 
@@ -237,6 +238,7 @@ mod tests {
                     .collect(),
                 api_key,
                 "claude",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|_| None,
             )
@@ -544,6 +546,12 @@ mod tests {
         &[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
     )]
     #[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
+    #[case::safeguards_need_the_dangerous_tool_use_beta(
+        json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[("anthropic-beta", "claude-code-20250219")],
+        &[("x-api-key", "k"), ("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03")]
+    )]
+    #[case::null_safeguards_need_no_beta(json!({"safeguards": null}), &[], &[("x-api-key", "k")])]
     fn request_headers_carry_the_anthropic_feature_betas(
         #[case] fields: serde_json::Value,
         #[case] forwarded: &[(&str, &str)],
@@ -616,6 +624,7 @@ mod tests {
             Vec::new(),
             None,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &record,
         );

litellm-rust/crates/llms/src/base_llm/messages/transformation.rs

@@ -1,4 +1,6 @@
-use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_llms_types::formats::messages::{
+    MessagesOptionalParams, MessagesRequest, MessagesResponse,
+};
 use litellm_router_types::LitellmParams;
 use serde_json::Value;
 
@@ -55,6 +57,7 @@ pub trait BaseMessagesConfig: Sync {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error>;
@@ -110,6 +113,7 @@ mod tests {
             headers: Headers,
             _api_key: Option<&str>,
             _model: &str,
+            _optional_params: &MessagesOptionalParams,
             _litellm_params: &LitellmParams,
             _env_lookup: &dyn Fn(&str) -> Option<String>,
         ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs

@@ -5,8 +5,8 @@ use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
 use litellm_core_utils::settings::resolve_non_empty;
 use litellm_http::request::{header_value, with_header, without_headers};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
-    providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
+    providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
 };
 use litellm_router_types::LitellmParams;
 use serde_json::Value;
@@ -15,7 +15,7 @@ use crate::{
     Error, ErrorDetail,
     anthropic::messages::{
         handler::shape_anthropic_messages_request,
-        transformation::{transform_messages_request, update_headers_with_anthropic_beta},
+        transformation::{transform_messages_request, update_headers_with_safeguards_beta},
     },
     base_llm::{
         auth::{AuthScheme, Headers, ValidatedEnvironment},
@@ -60,6 +60,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        optional_params: &MessagesOptionalParams,
         params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -67,11 +68,8 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
             .filter(|value| !value.trim().is_empty())
             .or_else(|| {
                 WORKSPACE_FIELDS.iter().find_map(|field| {
-                    params
-                        .extra
-                        .get(*field)?
-                        .as_str()
-                        .filter(|value| !value.trim().is_empty())
+                    workspace_value(optional_params.extra.get(*field))
+                        .or_else(|| workspace_value(params.extra.get(*field)))
                 })
             });
         let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS)
@@ -174,7 +172,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
             Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()),
             None => headers,
         };
-        update_headers_with_anthropic_beta(headers, request)
+        update_headers_with_safeguards_beta(headers, request, BetaProvider::Anthropic)
     }
 
     fn wire_body(&self, body: Value) -> Value {
@@ -182,6 +180,10 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
     }
 }
 
+fn workspace_value(value: Option<&Value>) -> Option<&str> {
+    value?.as_str().filter(|value| !value.trim().is_empty())
+}
+
 fn platform_region(
     base: Option<&str>,
     params: &LitellmParams,
@@ -260,6 +262,7 @@ mod tests {
                     vec![],
                     Some("key"),
                     "claude-test",
+                    &MessagesOptionalParams::default(),
                     &LitellmParams::default(),
                     &|_| None
                 )
@@ -280,6 +283,7 @@ mod tests {
                 ],
                 Some("explicit"),
                 "claude-test",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|name| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into()),
             )
@@ -345,7 +349,14 @@ mod tests {
             .get_complete_url(None, "claude-test", &params, false, &env)
             .unwrap();
         let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
-            .validate_environment(vec![], None, "claude-test", &params, &env)
+            .validate_environment(
+                vec![],
+                None,
+                "claude-test",
+                &MessagesOptionalParams::default(),
+                &params,
+                &env,
+            )
             .unwrap();
         assert_eq!(
             url,
@@ -356,6 +367,21 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case::safeguards(json!([{"type": "dangerous_tool_use"}]), Some("dangerous-tool-use-2026-09-03"))]
+    #[case::null_safeguards(Value::Null, None)]
+    fn safeguards_need_the_dangerous_tool_use_beta(
+        #[case] safeguards: Value,
+        #[case] expected: Option<&str>,
+    ) {
+        let request = serde_json::from_value(json!({"model": "claude-test", "max_tokens": 16, "messages": [], "safeguards": safeguards})).unwrap();
+        let headers = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.request_headers(vec![], &request);
+        assert_eq!(
+            header_value(&headers, "anthropic-beta").map(|value| value.to_string()),
+            expected.map(str::to_string)
+        );
+    }
+
     #[rstest]
     #[case::default(json!({}), false)]
     #[case::override_empty(json!({"claude_platform_unsupported_params": []}), true)]

litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs

@@ -16,7 +16,7 @@ use litellm_auth_aws::{
     resolve_bedrock_region,
 };
 use litellm_llms_types::formats::messages::{
-    MessagesRequest,
+    MessagesOptionalParams, MessagesRequest,
     streaming::{MessagesStreamEvent, MessagesStreamUsage},
 };
 use litellm_llms_types::providers::bedrock::{
@@ -146,6 +146,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        _optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -618,6 +619,7 @@ mod tests {
                 Vec::new(),
                 None,
                 "anthropic.claude-3",
+                &MessagesOptionalParams::default(),
                 &litellm_params,
                 &|_| None,
             )
@@ -658,6 +660,7 @@ mod tests {
                 vec![("authorization".into(), "Bearer forwarded".into())],
                 api_key,
                 "anthropic.claude-3",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &env,
             )

litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs

@@ -6,7 +6,7 @@ use litellm_auth_aws::bedrock_model_id_and_region;
 use litellm_core_utils::settings::resolve_non_empty;
 use litellm_http::request::{with_header, without_headers};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
     providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
 };
 use litellm_router_types::LitellmParams;
@@ -59,6 +59,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        _optional_params: &MessagesOptionalParams,
         params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -317,7 +318,14 @@ mod tests {
             "https://private.test/anthropic/v1/messages"
         );
         let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
-            .validate_environment(vec![], None, model, &params, &|_| None)
+            .validate_environment(
+                vec![],
+                None,
+                model,
+                &MessagesOptionalParams::default(),
+                &params,
+                &|_| None,
+            )
             .unwrap();
         assert!(
             matches!(environment.auth, AuthScheme::AwsSigV4 { region: actual, service: "bedrock", .. } if actual == region)
@@ -347,7 +355,14 @@ mod tests {
             .get_complete_url(None, "claude-test", &params, false, &|_| None)
             .unwrap();
         let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
-            .validate_environment(vec![], None, "claude-test", &params, &|_| None)
+            .validate_environment(
+                vec![],
+                None,
+                "claude-test",
+                &MessagesOptionalParams::default(),
+                &params,
+                &|_| None,
+            )
             .unwrap();
         assert_eq!(
             url,
@@ -401,6 +416,7 @@ mod tests {
                 ],
                 key,
                 "claude-test",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|name| match name {
                     "BEDROCK_MANTLE_API_KEY" => Some("mantle".into()),

litellm-rust/crates/llms/src/deepseek/messages/transformation.rs

@@ -104,6 +104,7 @@ impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -298,6 +299,7 @@ mod tests {
                 .collect(),
             api_key,
             "deepseek-v4-pro",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             env,
         )
@@ -451,4 +453,27 @@ mod tests {
             json!({"type": "enabled", "budget_tokens": 1024})
         );
     }
+
+    #[rstest]
+    #[case::without_client_betas(&[])]
+    #[case::with_client_betas(&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")])]
+    fn safeguards_and_client_betas_reach_deepseek_as_sent(#[case] client: &[(&str, &str)]) {
+        let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
+        let body = json!({
+            "model": "deepseek-v4-pro",
+            "max_tokens": 100,
+            "messages": [{"role": "user", "content": "hi"}],
+            "safeguards": safeguards
+        });
+        let headers: Headers = client
+            .iter()
+            .map(|(name, value)| (name.to_string(), value.to_string()))
+            .collect();
+        assert_eq!(
+            DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG
+                .request_headers(headers.clone(), &request_from(body.clone())),
+            headers
+        );
+        assert_eq!(transformed(body)["safeguards"], safeguards);
+    }
 }

litellm-rust/crates/llms/src/edenai/messages/transformation.rs

@@ -36,6 +36,7 @@ impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/minimax/messages/transformation.rs

@@ -42,6 +42,7 @@ impl BaseMessagesConfig for MinimaxMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -183,7 +184,14 @@ mod tests {
                 .flatten()
         };
         let environment = MINIMAX_MESSAGES_CONFIG
-            .validate_environment(vec![], key, "test-model", &Default::default(), &env)
+            .validate_environment(
+                vec![],
+                key,
+                "test-model",
+                &MessagesOptionalParams::default(),
+                &Default::default(),
+                &env,
+            )
             .unwrap();
         assert!(matches!(
             environment.auth,

litellm-rust/crates/llms/src/openrouter/messages/transformation.rs

@@ -34,6 +34,7 @@ impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/tencent/messages/transformation.rs

@@ -43,6 +43,7 @@ impl BaseMessagesConfig for TencentAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -229,7 +230,14 @@ mod tests {
                 .flatten()
         };
         let environment = TENCENT_MESSAGES_CONFIG
-            .validate_environment(vec![], key, "test-model", &Default::default(), &env)
+            .validate_environment(
+                vec![],
+                key,
+                "test-model",
+                &MessagesOptionalParams::default(),
+                &Default::default(),
+                &env,
+            )
             .unwrap();
         assert!(matches!(
             environment.auth,

litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs

@@ -114,6 +114,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
         headers: Headers,
         _api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         _env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -213,23 +214,14 @@ fn has_web_search_tool(tools: Option<&[Recognized<MessagesTool>]>) -> bool {
 }
 
 /// Python's Vertex `validate_anthropic_messages_environment` beta set: the shared feature
-/// betas under Vertex's policy, plus the web search and safeguards betas Vertex needs spelled
-/// out.
+/// betas under Vertex's policy, plus the web search beta Vertex needs spelled out.
 fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
     let params = &request.params;
     provider_feature_betas(request, BetaProvider::VertexAi).union(
-        [
-            has_web_search_tool(params.tools.as_deref())
-                .then_some(AnthropicBeta::WebSearch20250305),
-            params
-                .extra
-                .get("safeguards")
-                .is_some_and(|value| !value.is_null())
-                .then_some(AnthropicBeta::DangerousToolUse20260903),
-        ]
-        .into_iter()
-        .flatten()
-        .collect(),
+        has_web_search_tool(params.tools.as_deref())
+            .then_some(AnthropicBeta::WebSearch20250305)
+            .into_iter()
+            .collect(),
     )
 }
 
@@ -448,6 +440,7 @@ mod tests {
                     .collect(),
                 Some("ignored-api-key"),
                 "claude-sonnet-4-5",
+                &MessagesOptionalParams::default(),
                 &params(Some("proj"), None),
                 &no_env,
             )

litellm-rust/crates/python-bridge/src/routes/responses/host.rs

@@ -1,22 +1,41 @@
 use std::convert::Infallible;
 
 use crate::routes::inference::InferenceHost;
-use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned};
-use litellm_inference_responses::{Error, route::Responses, types::ResponsesCall};
+use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
+use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned, from_py};
+use litellm_inference_responses::{
+    Error,
+    ids::{self, IdScope},
+    route::Responses,
+    types::ResponsesCall,
+};
 use pyo3::{
     gc::{PyTraverseError, PyVisit},
     prelude::*,
     types::PyDict,
 };
 
-pub(super) struct ResponsesPythonHost(pub InferenceHost);
+pub(super) struct ResponsesPythonHost {
+    host: InferenceHost,
+    ids: IdScope,
+}
+
+impl ResponsesPythonHost {
+    pub fn new(host: InferenceHost) -> Self {
+        Self {
+            host,
+            ids: IdScope::default(),
+        }
+    }
+}
 
 pub(super) fn project(
     host: &mut InferenceHost,
     py: Python<'_>,
     arguments: &Bound<'_, PyDict>,
 ) -> PyResult<ResponsesCall> {
-    let call = host.project(py, arguments, "input")?;
+    let mut call = host.project(py, arguments, "input")?;
+    ids::restore_input(&mut call.input);
     let optional_params = call
         .params
         .into_iter()
@@ -50,6 +69,39 @@ pub(super) fn project(
     })
 }
 
+/// The provider and deployment Python encodes into the ids it returns: the resolved provider and
+/// `litellm_metadata.model_info.id`
+fn id_scope(
+    host: &InferenceHost,
+    py: Python<'_>,
+    arguments: &Bound<'_, PyDict>,
+    call: &ResponsesCall,
+) -> PyResult<IdScope> {
+    let metadata = host.argument(py, arguments, "litellm_metadata")?;
+    let encrypted_content_affinity = match &metadata {
+        Some(metadata) => metadata
+            .call_method1("get", ("encrypted_content_affinity_enabled",))?
+            .is_truthy()?,
+        None => false,
+    };
+    let metadata = metadata
+        .map(|value| from_py::<serde_json::Value>(&value))
+        .transpose()?
+        .unwrap_or_default();
+    let model_id = match metadata.pointer("/model_info/id") {
+        None | Some(serde_json::Value::Null) => None,
+        Some(serde_json::Value::String(id)) => Some(id.clone()),
+        Some(other) => Some(other.to_string()),
+    };
+    Ok(IdScope {
+        provider: get_custom_llm_provider(&call.model, call.custom_llm_provider.as_deref())
+            .map(|resolved| resolved.custom_llm_provider.to_owned())
+            .or_else(|| call.custom_llm_provider.clone()),
+        model_id,
+        encrypted_content_affinity,
+    })
+}
+
 impl PythonBinding for ResponsesPythonHost {
     type Protocol = Responses;
     type Failure = PyErr;
@@ -59,7 +111,8 @@ impl PythonBinding for ResponsesPythonHost {
         py: Python<'_>,
         arguments: &Bound<'_, PyDict>,
     ) -> Result<ResponsesCall, InvokeError<Error>> {
-        let call = project(&mut self.0, py, arguments).map_err(InvokeError::Python)?;
+        let call = project(&mut self.host, py, arguments).map_err(InvokeError::Python)?;
+        self.ids = id_scope(&self.host, py, arguments, &call).map_err(InvokeError::Python)?;
         if call
             .optional_params
             .get("stream")
@@ -77,7 +130,9 @@ impl PythonBinding for ResponsesPythonHost {
         py: Python<'_>,
         response: <Responses as litellm_host::protocol::Protocol>::Response,
     ) -> PyResult<Py<PyAny>> {
-        self.0.response(py, &response)
+        let mut response = response;
+        ids::encode_response(&mut response, &self.ids);
+        self.host.response(py, &response)
     }
 
     fn encode_stream_head(
@@ -103,7 +158,7 @@ impl PythonBinding for ResponsesPythonHost {
     }
 
     fn map_error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
-        self.0.error(py, error)
+        self.host.error(py, error)
     }
     fn host_error(error: &PyErr) -> Error {
         Error::InvalidRequest(error.to_string().into())
@@ -122,9 +177,9 @@ impl PythonHostCalls<Responses> for ResponsesPythonHost {
 
 impl PythonOwned for ResponsesPythonHost {
     fn close(&mut self, _: Python<'_>) {
-        self.0.close();
+        self.host.close();
     }
     fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
-        self.0.traverse(visit)
+        self.host.traverse(visit)
     }
 }

litellm-rust/crates/python-bridge/src/routes/responses/mod.rs

@@ -62,7 +62,7 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py
         ));
     }
     let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE);
-    run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost(host))
+    run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost::new(host))
 }
 
 #[pyfunction]

litellm/__init__.py

@@ -7,6 +7,7 @@ _apply_process_defaults()
 ### INIT VARIABLES #########################
 import threading
 import os
+import datetime
 
 from collections.abc import Mapping, Sequence
 from typing import (
@@ -62,7 +63,6 @@ from litellm.constants import (
     modelscope_models,
     empower_models,
     together_ai_models,
-    baseten_models,
     WANDB_MODELS,
     REPEATED_STREAMING_CHUNK_LIMIT,
     request_timeout,
@@ -624,6 +624,7 @@ bedrock_models: Set = set()
 bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS)
 fal_ai_models: Set = set()
 fireworks_ai_models: Set = set()
+baseten_models: Set[str] = set()
 fireworks_ai_embedding_models: Set = set()
 deepinfra_models: Set = set()
 perplexity_models: Set = set()
@@ -676,6 +677,7 @@ qwen_ai_platform_models: Set = set()
 moonshot_models: Set = set()
 publicai_models: Set = set()
 darkbloom_models: Set = set()
+openinfer_models: Final[Set[str]] = set()
 v0_models: Set = set()
 morph_models: Set = set()
 lambda_ai_models: Set = set()
@@ -741,8 +743,18 @@ def is_openai_finetune_model(key: str) -> bool:
     return key.startswith("ft:") and not key.count(":") > 1
 
 
+def _is_past_deprecation_date(value: Mapping[str, object]) -> bool:
+    deprecation_date: Final = value.get("deprecation_date")
+    if not isinstance(deprecation_date, str):
+        return False
+    try:
+        return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today()
+    except ValueError:
+        return False
+
+
 def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
-    for key, value in model_cost_map.items():
+    for key, value in tuple(model_cost_map.items()):
         if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
             open_ai_chat_completion_models.add(key)
         elif value.get("litellm_provider") == "text-completion-openai":
@@ -841,6 +853,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
             if "-to-" not in key and "fireworks-ai-default" not in key:
                 fireworks_ai_models.add(key)
+        elif value.get("litellm_provider") == "baseten":
+            if _is_past_deprecation_date(value):
+                baseten_models.discard(key)
+            else:
+                baseten_models.add(key)
         elif value.get("litellm_provider") == "fireworks_ai-embedding-models":
             # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
             if "-to-" not in key:
@@ -945,6 +962,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             publicai_models.add(key)
         elif value.get("litellm_provider") == "darkbloom":
             darkbloom_models.add(key)
+        elif value.get("litellm_provider") == "openinfer":
+            openinfer_models.add(key)
         elif value.get("litellm_provider") == "v0":
             v0_models.add(key)
         elif value.get("litellm_provider") == "morph":
@@ -1113,6 +1132,7 @@ model_list = list(
     | moonshot_models
     | publicai_models
     | darkbloom_models
+    | openinfer_models
     | v0_models
     | morph_models
     | lambda_ai_models
@@ -1229,6 +1249,7 @@ def _build_models_by_provider() -> dict:
         "moonshot": moonshot_models,
         "publicai": publicai_models,
         "darkbloom": darkbloom_models,
+        "openinfer": openinfer_models,
         "v0": v0_models,
         "morph": morph_models,
         "lambda_ai": lambda_ai_models,

litellm/_logging.py

@@ -1000,6 +1000,8 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
         lg.handlers.clear()  # remove any existing handlers
         lg.addHandler(handler)  # add JSON formatter handler
         lg.propagate = False  # prevent bubbling to parent/root
+    if (configured_level := os.getenv("LITELLM_LOG")) is not None:
+        logging.getLogger().setLevel(resolve_log_level(configured_level))
 
 
 def _get_uvicorn_json_log_config():

litellm/anthropic_beta_headers_config.json

@@ -220,36 +220,36 @@
   },
   "databricks": {
     "advisor-tool-2026-03-01": null,
-    "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
+    "advanced-tool-use-2025-11-20": null,
     "bash_20241022": null,
     "bash_20250124": null,
-    "code-execution-2025-08-25": "code-execution-2025-08-25",
+    "code-execution-2025-08-25": null,
     "compact-2026-01-12": "compact-2026-01-12",
     "computer-use-2025-01-24": "computer-use-2025-01-24",
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
-    "dangerous-tool-use-2026-09-03": null,
+    "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
-    "fast-mode-2026-02-01": "fast-mode-2026-02-01",
-    "files-api-2025-04-14": "files-api-2025-04-14",
+    "fast-mode-2026-02-01": null,
+    "files-api-2025-04-14": null,
     "structured-output-2024-03-01": null,
     "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
-    "mcp-client-2025-11-20": "mcp-client-2025-11-20",
-    "mcp-client-2025-04-04": "mcp-client-2025-04-04",
+    "mcp-client-2025-11-20": null,
+    "mcp-client-2025-04-04": null,
     "mcp-servers-2025-12-04": null,
-    "oauth-2025-04-20": "oauth-2025-04-20",
+    "oauth-2025-04-20": null,
     "output-128k-2025-02-19": "output-128k-2025-02-19",
     "per-turn-control-2026-07-01": null,
-    "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
-    "skills-2025-10-02": "skills-2025-10-02",
+    "prompt-caching-scope-2026-01-05": null,
+    "skills-2025-10-02": null,
     "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
     "text_editor_20241022": null,
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
-    "web-fetch-2025-09-10": "web-fetch-2025-09-10",
+    "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": "web-search-2025-03-05"
   }
 }

litellm/anthropic_beta_headers_manager.py

@@ -22,17 +22,28 @@ Environment Variables:
 - LITELLM_ANTHROPIC_BETA_HEADERS_URL: Custom URL for remote config (optional)
 """
 
+import asyncio
 import json
 import os
+import threading
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor
+from functools import cache
 from importlib.resources import files
-from typing import Final
+from typing import Final, TypeAlias
 
 import httpx
+from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
 
 # Cache for the loaded configuration
 _BETA_HEADERS_CONFIG: dict | None = None
+_BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock()
+_BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config")
+
+BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
+_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
 
 
 class GetAnthropicBetaHeadersConfig:
@@ -174,13 +185,27 @@ def _load_beta_headers_config() -> dict:
     if _BETA_HEADERS_CONFIG is not None:
         return _BETA_HEADERS_CONFIG
 
-    # Get the URL from environment or use default
-    from litellm import anthropic_beta_headers_url
+    with _BETA_HEADERS_CONFIG_LOCK:
+        if _BETA_HEADERS_CONFIG is not None:
+            return _BETA_HEADERS_CONFIG
 
-    _BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
-    verbose_logger.debug("Loaded and cached beta headers config")
+        # Get the URL from environment or use default
+        from litellm import anthropic_beta_headers_url
 
-    return _BETA_HEADERS_CONFIG
+        _BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
+        verbose_logger.debug("Loaded and cached beta headers config")
+
+        return _BETA_HEADERS_CONFIG
+
+
+async def aload_beta_headers_config() -> None:
+    if _BETA_HEADERS_CONFIG is None:
+        await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config)
+
+
+@cache
+def _bundled_beta_headers_config() -> BetaHeadersConfig:
+    return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config())
 
 
 def reload_beta_headers_config() -> dict:
@@ -288,6 +313,14 @@ def is_beta_header_supported(
     return beta_header in provider_mapping and provider_mapping[beta_header] is not None
 
 
+def is_beta_header_supported_by_bundled_config(beta_header: str, provider: str) -> bool:
+    config: Final = _bundled_beta_headers_config()
+    aliases: Final = config.get("provider_aliases", {})
+    canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
+    provider_mapping: Final = config.get(canonical_provider, {})
+    return isinstance(provider_mapping, Mapping) and provider_mapping.get(beta_header) is not None
+
+
 def get_provider_beta_header(
     anthropic_beta_header: str,
     provider: str,

litellm/batches/batch_utils.py

@@ -794,11 +794,17 @@ def _batch_response_was_successful(
     message batch results lines report ``result.type == "succeeded"``; Bedrock
     batch output lines report ``modelOutput`` (and no ``error``).
     """
-    if custom_llm_provider == "anthropic":
-        return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
     if custom_llm_provider == "bedrock":
         return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None
-    _response: Final[dict[str, object]] = cast(  # cast-ok: batch response bodies come from provider JSON
-        dict[str, object], batch_job_output_file.get("response", None) or {}
+    envelope_key: Final = "result" if custom_llm_provider == "anthropic" else "response"
+    envelope: Final = batch_job_output_file.get(envelope_key)
+    if not isinstance(envelope, Mapping):
+        if envelope is not None:
+            verbose_logger.warning("batch output %s must be an object, treating the line as unsuccessful", envelope_key)
+        return False
+    response: Final = cast(Mapping[object, object], envelope)  # cast-ok: mapping shape validated above
+    return (
+        response.get("type") == "succeeded"
+        if custom_llm_provider == "anthropic"
+        else response.get("status_code") == 200
     )
-    return _response.get("status_code", None) == 200

litellm/caching/caching_handler.py

@@ -346,6 +346,7 @@ class LLMCachingHandler:
 
             if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function):
                 verbose_logger.debug("Checking Async Cache")
+                logging_obj.reset_caching_details()
                 cached_result = await self._retrieve_from_cache(
                     call_type=call_type,
                     kwargs=kwargs,
@@ -357,7 +358,7 @@ class LLMCachingHandler:
                     verbose_logger.debug("Cache Hit!")
                     cache_hit: Final = True
                     end_time: Final = datetime.datetime.now()
-                    model, custom_llm_provider, _, _ = litellm.get_llm_provider(
+                    provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider(
                         model=model,
                         custom_llm_provider=kwargs.get("custom_llm_provider", None),
                         api_base=kwargs.get("api_base", None),
@@ -366,7 +367,7 @@ class LLMCachingHandler:
                     cache_duration_ms: Final = (cache_check_end_time - cache_check_start_time) * 1000
                     self._update_litellm_logging_obj_environment(
                         logging_obj=logging_obj,
-                        model=model,
+                        model=provider_model,
                         kwargs=kwargs,
                         cached_result=cached_result,
                         is_async=True,
@@ -381,7 +382,7 @@ class LLMCachingHandler:
                         call_type=call_type,
                         kwargs=kwargs,
                         logging_obj=logging_obj,
-                        model=model,
+                        model=provider_model,
                         custom_llm_provider=kwargs.get("custom_llm_provider", None),
                         args=args,
                     )
@@ -467,6 +468,7 @@ class LLMCachingHandler:
                 new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
             self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
             print_verbose("Checking Sync Cache")
+            logging_obj.reset_caching_details()
             with response_cache_phase("get"):
                 cached_result = litellm.cache.get_cache(**new_kwargs)
             if is_response_without_output(cached_result):

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -41,7 +41,11 @@ from litellm.responses.sse_output_recovery import (
     record_output_item_chunk,
     record_output_text_chunk,
 )
-from litellm.responses.utils import ResponsesAPIRequestUtils, normalize_responses_api_stream_options
+from litellm.responses.utils import (
+    ResponsesAPIRequestUtils,
+    auto_reasoning_summary,
+    normalize_responses_api_stream_options,
+)
 from litellm.types.llms.openai import (
     ChatCompletionAnnotation,
     ChatCompletionReasoningItem,
@@ -176,6 +180,16 @@ def _without_audio_input_parts(input_items: list[object]) -> list[object]:
     return [_without_audio_input_parts_in_item(item) for item in input_items]
 
 
+def _effort_with_requested_summary(nested_effort: object, top_level_effort: object) -> object:
+    match top_level_effort, nested_effort:
+        case {"summary": str() as summary}, str() as effort:
+            return {"effort": effort, "summary": summary}
+        case {"summary": str() as summary}, Mapping() as reasoning if "summary" not in reasoning:
+            return {**reasoning, "summary": summary}
+        case _:
+            return nested_effort
+
+
 def _served_provider(litellm_params: Mapping[str, object]) -> str | None:
     custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
     return custom_llm_provider if isinstance(custom_llm_provider, str) else None
@@ -953,6 +967,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         optional_params: dict,
         responses_api_request: "ResponsesAPIOptionalRequestParams",
         drop_params: bool = False,
+        model: str | None = None,
+        custom_llm_provider: str | None = None,
     ) -> None:
         """Map optional_params into responses_api_request (mutates in place)."""
         for key, value in optional_params.items():
@@ -984,7 +1000,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             elif key == "previous_response_id":
                 responses_api_request["previous_response_id"] = value
             elif key == "reasoning_effort":
-                responses_api_request["reasoning"] = self.map_reasoning_effort(value)
+                responses_api_request["reasoning"] = self.map_reasoning_effort(value, model, custom_llm_provider)
             elif key == "web_search_options":
                 self._add_web_search_tool(responses_api_request, value)
 
@@ -1100,6 +1116,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             optional_params,
             responses_api_request,
             drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
+            model=model,
+            custom_llm_provider=_served_provider(litellm_params),
         )
 
         stream: Final = optional_params.get("stream") or litellm_params.get("stream", False)
@@ -1705,14 +1723,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         # Extract supported params from extra_body and merge into optional_params
         extra_body_copy: Final = extra_body.copy()
+        top_level_effort: Final = cast(object, optional_params.get("reasoning_effort"))  # cast-ok: upcast
         for key, value in extra_body_copy.items():
             if key in supported_responses_api_params:
                 # Prefer extra_body value if it exists (may have more complete info like summary in reasoning_effort)
-                optional_params[key] = extra_body.pop(key)
+                extra_body.pop(key)
+                optional_params[key] = (
+                    _effort_with_requested_summary(cast(object, value), top_level_effort)  # cast-ok: upcast
+                    if key == "reasoning_effort"
+                    else value
+                )
 
         return optional_params
 
-    def map_reasoning_effort(self, reasoning_effort: object) -> Reasoning:
+    def map_reasoning_effort(
+        self, reasoning_effort: object, model: str | None = None, custom_llm_provider: str | None = None
+    ) -> Reasoning:
         # If dict is passed, convert it directly to Reasoning object
         if isinstance(reasoning_effort, dict):
             return Reasoning(
@@ -1726,7 +1752,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         )
 
         return (
-            Reasoning(effort=reasoning_effort, summary="detailed")
+            Reasoning(
+                effort=reasoning_effort,
+                summary="detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider),
+            )
             if auto_summary_enabled
             else Reasoning(effort=reasoning_effort)
         )

litellm/constants.py

@@ -40,6 +40,14 @@ class ServerStreamingClassification(str, Enum):
 SERVER_STREAMING_CLASSIFICATION_MARKER: Final = ServerStreamingClassification.MARKER
 DEFER_PYDANTIC_BUILD: Final = os.getenv("DEFER_PYDANTIC_BUILD", "true") in ("true", "1", "on")
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
+KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS: Final = get_env_float(
+    "KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS", 5.0
+)
+KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS: Final = get_env_float(
+    "KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS", 300.0
+)
+KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS: Final = 2.0
+KUBERNETES_POD_ROUTING_KEY: Final = "kubernetes_pod_routing"
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
 AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"})
 ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5)
@@ -244,6 +252,13 @@ MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_PER_USER_TOKE
 # MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers.
 MCP_CLIENT_TIMEOUT: Final = get_env_float("LITELLM_MCP_CLIENT_TIMEOUT", 60.0)
 MCP_TOOL_LISTING_TIMEOUT: Final = get_env_float("LITELLM_MCP_TOOL_LISTING_TIMEOUT", 30.0)
+MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES: Final = get_env_int("LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES", 3)
+MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS: Final = get_env_float(
+    "LITELLM_MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS", 0.5
+)
+MCP_UPSTREAM_RATE_LIMIT_MAX_DELAY_SECONDS: Final = get_env_float(
+    "LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_DELAY_SECONDS", 5.0
+)
 MCP_METADATA_TIMEOUT: Final = get_env_float("LITELLM_MCP_METADATA_TIMEOUT", 10.0)
 MCP_HEALTH_CHECK_TIMEOUT: Final = get_env_float("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", 10.0)
 AGENT_HEALTH_CHECK_TIMEOUT_SECONDS: Final = get_env_float("LITELLM_AGENT_HEALTH_CHECK_TIMEOUT", 5.0)
@@ -1030,6 +1045,7 @@ openai_compatible_endpoints: Final[list[str]] = [
     "https://api.prisminference.com/v1",
     "https://api.reka.ai/v1",
     "https://gigachat.devices.sberbank.ru/api/v1",
+    "https://api.openinfer.ai/v1",
 ]
 
 
@@ -1098,6 +1114,7 @@ openai_compatible_providers: Final[list] = [
     "docker_model_runner",
     "ragflow",
     "pinstripes",  # Pinstripes - JSON-configured provider
+    "openinfer",
     "darkbloom",
     "meta",  # Meta Model API (Muse Spark) - JSON-configured provider
     "cognition",
@@ -1254,14 +1271,6 @@ together_ai_models: Final[set] = set(
 # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
 
 
-baseten_models: Final[set] = set(
-    [
-        "qvv0xeq",
-        "q841o8w",
-        "31dxrj3",
-    ]
-)  # FALCON 7B  # WizardLM  # Mosaic ML
-
 featherless_ai_models: Final[set] = set(
     [
         "featherless-ai/Qwerky-72B",
@@ -1631,6 +1640,7 @@ OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_to
 CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
+VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
 ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"

litellm/cost_calculator.py

@@ -1103,6 +1103,8 @@ def get_usage_object(
 
     if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
         return None if completion_response.usage is None else _decisions_usage(completion_response.usage)
+    if isinstance(completion_response, HttpxBinaryResponseContent):
+        return completion_response.usage
     if usage_obj is None:
         return None
     if isinstance(usage_obj, Usage):
@@ -3113,14 +3115,19 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
         return response_ids[-1] if response_ids else None
 
     @staticmethod
-    def create_logging_object(results: Sequence[Mapping[str, object]]) -> LiteLLMRealtimeStreamLoggingObject:
+    def create_logging_object(
+        results: Sequence[Mapping[str, object]], litellm_model_name: str | None = None
+    ) -> LiteLLMRealtimeStreamLoggingObject:
         tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(results)
-        return LiteLLMRealtimeStreamLoggingObject(
+        logging_object: Final = LiteLLMRealtimeStreamLoggingObject(
             id=ResponsesWebSocketTokenUsageProcessor.response_id_from_responses_ws_results(results),
             usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(results),
             results=cast(OpenAIRealtimeStreamList, list(results)),  # cast-ok: raw WS frames, unvalidated field
             service_tier=next(iter(tier_partition)) if len(tier_partition) == 1 else None,
         )
+        if litellm_model_name is not None:
+            logging_object.hidden_params = {"litellm_model_name": litellm_model_name}
+        return logging_object
 
 
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"

litellm/decisions/call.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from functools import partial
@@ -6,6 +7,7 @@ from typing import Final, Literal, TypeAlias
 
 import httpx
 from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
+from pydantic_core import to_jsonable_python
 from typing_extensions import assert_never
 
 import litellm
@@ -56,6 +58,7 @@ from litellm.types.decisions import (
     OpenAIDecisionResponse,
     UnsupportedDecisionsRequest,
 )
+from litellm.types.llms.openai import ChatCompletionUserMessage
 from litellm.types.utils import LlmProviders, ProviderSpecificHeader
 from litellm.utils import ProviderConfigManager, get_model_cost_key
 
@@ -415,6 +418,30 @@ def _outbound_headers(
     return MappingProxyType({**(extra_headers or {}), **forwarded})
 
 
+_LOGGED_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+
+
+def _jsonable_fallback(value: object) -> object:
+    return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value)
+
+
+def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage:
+    caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python(
+        to_jsonable_python(caller_request, fallback=_jsonable_fallback)
+    )
+    sent_fields: Final = json.dumps(
+        dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {}))
+    )
+    return ChatCompletionUserMessage(role="user", content=sent_fields)
+
+
+def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None:
+    """Record every request field the caller sent as the logged `messages`, before any validation."""
+    logging_obj: Final = kwargs.get("litellm_logging_obj")
+    if isinstance(logging_obj, LiteLLMLoggingObj):
+        logging_obj.update_messages([_logged_request_message(caller_request)])
+
+
 def prepare_call(
     *,
     model: str,

litellm/decisions/main.py

@@ -17,6 +17,7 @@ from litellm.decisions.call import (
     SystemOneQuestions,
     SystemOneRequestFields,
     asend,
+    log_caller_request,
     prepare_call,
     reject_images_next_to_input,
     send,
@@ -78,6 +79,16 @@ def _prepare_decisions_call(
     extra_headers: Mapping[str, str] | None,
     kwargs: Mapping[str, object],
 ) -> DecisionsCall:
+    log_caller_request(
+        kwargs,
+        {
+            "input": decision_input,
+            "questions": questions,
+            "audio": audio,
+            "videos": videos,
+            **{key: kwargs[key] for key in _SYSTEMONE_FIELDS if key in kwargs},
+        },
+    )
     return prepare_call(
         model=model,
         fields=_request_fields(model, decision_input, questions, audio, videos, kwargs),

litellm/endpoints/speech/speech_to_completion_bridge/handler.py

@@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, Final
 
 from typing_extensions import TypedDict
 
+from litellm._internal_context import internal_sub_call
+
 if TYPE_CHECKING:
     from litellm import LiteLLMLoggingObj
     from litellm.types.llms.openai import HttpxBinaryResponseContent
@@ -111,9 +113,10 @@ class SpeechToCompletionBridgeHandler:
             voice=voice,
         )
 
-        result: Final = completion(
-            **request_data,
-        )
+        with internal_sub_call():
+            result: Final = completion(
+                **request_data,
+            )
 
         requested_response_format: Final = optional_params.get("response_format")
         if isinstance(result, ModelResponse):

litellm/endpoints/speech/speech_to_completion_bridge/transformation.py

@@ -5,11 +5,12 @@ from typing import TYPE_CHECKING, Final, cast
 from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
+from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_param
 
 if TYPE_CHECKING:
     from litellm import Logging as LiteLLMLoggingObj
     from litellm.types.llms.openai import ChatCompletionUserMessage, HttpxBinaryResponseContent
-    from litellm.types.utils import ModelResponse
+    from litellm.types.utils import ModelResponse, Usage
 
 
 def _completion_response_cost(model_response: "ModelResponse") -> float | None:
@@ -20,6 +21,13 @@ def _completion_response_cost(model_response: "ModelResponse") -> float | None:
     return response_cost if isinstance(response_cost, float) else None
 
 
+def _completion_usage(model_response: "ModelResponse") -> "Usage | None":
+    from litellm.types.utils import Usage
+
+    usage: Final = getattr(model_response, "usage", None)
+    return usage if isinstance(usage, Usage) else None
+
+
 GEMINI_TTS_CHAT_AUDIO_FORMAT: Final = "pcm16"
 GEMINI_TTS_RAW_RESPONSE_FORMAT: Final = "pcm"
 GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS: Final = frozenset({"wav", GEMINI_TTS_RAW_RESPONSE_FORMAT})
@@ -179,4 +187,9 @@ class SpeechToCompletionBridgeTransformationHandler:
         )
         binary_response: Final = HttpxBinaryResponseContent(response)
         binary_response.set_response_cost(_completion_response_cost(model_response))
+        binary_response.usage = _completion_usage(model_response)
+        cached_hidden_params: Final = get_hidden_params(model_response) if served_from_cache(model_response) else None
+        if cached_hidden_params is not None:
+            set_hidden_param(binary_response, "cache_hit", True)
+            set_hidden_param(binary_response, "cache_key", cached_hidden_params.get("cache_key"))
         return binary_response

litellm/fine_tuning/main.py

@@ -173,6 +173,7 @@ def create_fine_tuning_job(
     try:
         _is_async: Final = kwargs.pop("acreate_fine_tuning_job", False) is True
         optional_params: Final = GenericLiteLLMParams(**kwargs)
+        provider_model: Final = model.removeprefix(f"{custom_llm_provider}/")
 
         # handle hyperparameters
         hyperparameters = hyperparameters or {}  # original hyperparameters
@@ -218,7 +219,7 @@ def create_fine_tuning_job(
             )
 
             create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
-                model,
+                provider_model,
                 training_file,
                 _oai_hyperparameters,
                 suffix,
@@ -263,7 +264,7 @@ def create_fine_tuning_job(
             extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams)
 
             create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
-                model,
+                provider_model,
                 training_file,
                 _oai_hyperparameters,
                 suffix,
@@ -298,7 +299,7 @@ def create_fine_tuning_job(
             response = vertex_fine_tuning_apis_instance.create_fine_tuning_job(
                 _is_async=_is_async,
                 create_fine_tuning_job_data=_build_fine_tuning_job_data(
-                    model,
+                    provider_model,
                     training_file,
                     _oai_hyperparameters,
                     suffix,

litellm/google_genai/main.py

@@ -16,6 +16,7 @@ from litellm.llms.base_llm.google_genai.transformation import (
     BaseGoogleGenAIGenerateContentConfig,
 )
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import CallTypes
@@ -194,12 +195,14 @@ class GenerateContentHelper:
         if litellm_logging_obj is None:
             raise ValueError("litellm_logging_obj is required, but got None")
 
+        vertex_location: Final = VertexBase.explicit_vertex_ai_location(dict(litellm_params))
         litellm_logging_obj.update_from_kwargs(
             kwargs=kwargs,
             model=model,
             optional_params=dict(generate_content_config_dict),
             litellm_params={
                 "litellm_call_id": litellm_call_id,
+                **({"vertex_location": vertex_location} if vertex_location else {}),
             },
             custom_llm_provider=custom_llm_provider,
         )

litellm/integrations/custom_guardrail.py

@@ -1139,6 +1139,17 @@ class CustomGuardrail(CustomLogger):
         """
         return False
 
+    def supports_structured_messages(self) -> bool:
+        """Whether this guardrail scans every row of ``structured_messages`` and returns texts that agree with them.
+
+        Guardrails that scan only part of the rows (the latest user turn, the
+        last row, the turns after the last assistant turn, or no tool results),
+        derive their returned texts from the rows, or cannot rewrite rows whose
+        content is a list of parts, override this to return False. Translation handlers that can rewrite the request from ``texts``
+        alone, like Bedrock Converse, then hand them only ``texts``.
+        """
+        return True
+
     def _suppressed_by_auto_router_compression(self) -> bool:
         """True when an auto router's own compression policy suppresses this guardrail.
 

litellm/integrations/custom_logger.py

@@ -912,9 +912,11 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         This is useful for logging payloads that contain sensitive information.
         """
         import litellm
-        from litellm import Choices, Message, ModelResponse
         from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS
-        from litellm.litellm_core_utils.redact_messages import redacted_litellm_params
+        from litellm.litellm_core_utils.redact_messages import (
+            redacted_litellm_params,
+            redacted_standard_logging_payload,
+        )
 
         turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False)
         excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
@@ -940,41 +942,14 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
             if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS)
         }
 
-        # Handle turn_off_message_logging - redact messages and responses (if not already excluded)
-        if turn_off_message_logging and not self.redacts_messages_itself():
-            redacted_str: Final = "redacted-by-litellm"
-
-            if "messages" not in (excluded_fields or ()) and standard_logging_object_copy.get("messages") is not None:
-                standard_logging_object_copy["messages"] = [Message(content=redacted_str).model_dump()]
-
-            if "response" not in (excluded_fields or ()) and standard_logging_object_copy.get("response") is not None:
-                response: Final = standard_logging_object_copy["response"]
-                # Check if this is a ResponsesAPIResponse (has "output" field)
-                if isinstance(response, dict) and "output" in response:
-                    # Make a copy to avoid modifying the original
-                    from copy import deepcopy
-
-                    response_copy: Final = deepcopy(response)
-                    # Redact content in output array
-                    if isinstance(response_copy.get("output"), list):
-                        for output_item in response_copy["output"]:
-                            if isinstance(output_item, dict) and "content" in output_item:
-                                if isinstance(output_item["content"], list):
-                                    # Redact text in content items
-                                    for content_item in output_item["content"]:
-                                        if isinstance(content_item, dict) and "text" in content_item:
-                                            content_item["text"] = redacted_str
-                    standard_logging_object_copy["response"] = response_copy
-                else:
-                    # Standard ModelResponse format
-                    model_response: Final = ModelResponse(choices=[Choices(message=Message(content=redacted_str))])
-                    model_response_dict: Final = model_response.model_dump()
-                    standard_logging_object_copy["response"] = model_response_dict
-
         return {
             **model_call_details,
             **redacted_params,
-            "standard_logging_object": standard_logging_object_copy,
+            "standard_logging_object": (
+                redacted_standard_logging_payload(standard_logging_object_copy)
+                if turn_off_message_logging and not self.redacts_messages_itself()
+                else standard_logging_object_copy
+            ),
         }
 
     async def get_proxy_server_request_from_cold_storage_with_object_key(

litellm/integrations/opentelemetry.py

@@ -6,12 +6,14 @@ from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import datetime
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
+from typing import TYPE_CHECKING, Any, Final, cast
 
 from pydantic import ConfigDict, TypeAdapter
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.constants import KUBERNETES_POD_ROUTING_KEY
 from litellm.integrations._types.open_inference import (
     OpenInferenceSpanKindValues,
 )
@@ -49,6 +51,7 @@ from litellm.types.utils import (
     Function,
     LLMResponseTypes,
     StandardCallbackDynamicParams,
+    StandardLoggingMetadata,
     StandardLoggingPayload,
 )
 
@@ -106,6 +109,14 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
 # Cap on credential-scoped providers held at once; each one owns an exporter thread.
 _MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
 
+
+class _CacheHitView(TypedDict, total=False):
+    cache_hit: ReadOnly[object]
+    saved_cache_cost: ReadOnly[object]
+
+
+_CACHE_HIT_VIEW: Final = TypeAdapter(_CacheHitView)
+
 # Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
 _PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
 
@@ -2411,9 +2422,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             #############################################
             ############ LLM CALL METADATA ##############
             #############################################
-            metadata: Final = standard_logging_payload["metadata"]
+            metadata: Final = cast(  # cast-ok: payload metadata follows the standard logging contract
+                StandardLoggingMetadata, standard_logging_payload["metadata"]
+            )
             for key, value in metadata.items():
-                self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value)
+                self.safe_set_attribute(
+                    span=span,
+                    key=f"metadata.{key}",
+                    value=safe_dumps(value) if key == KUBERNETES_POD_ROUTING_KEY and value is not None else value,
+                )
             decision: Final = metadata.get("routing_decision")
             span.set_attributes(routing_decision_attributes(decision))
 
@@ -2434,6 +2451,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                 litellm_params=litellm_params,
             )
             # Cost breakdown tracking
+            cache_view: Final = _CACHE_HIT_VIEW.validate_python(standard_logging_payload)
+            cache_hit: Final = cache_view.get("cache_hit") is True
             cost_breakdown: Final[CostBreakdown | None] = standard_logging_payload.get("cost_breakdown")
             if cost_breakdown:
                 for key, value in cost_breakdown.items():
@@ -2443,6 +2462,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                             key=f"gen_ai.cost.{key}",
                             value=value,
                         )
+            self.safe_set_attribute(span=span, key=LiteLLM.CACHE_HIT, value=cache_hit)
+            if cache_hit:
+                saved_cache_cost: Final = cache_view.get("saved_cache_cost")
+                if isinstance(saved_cache_cost, (int, float)):
+                    self.safe_set_attribute(span=span, key=LiteLLM.SAVED_CACHE_COST, value=saved_cache_cost)
             #############################################
             ########## LLM Request Attributes ###########
             #############################################

litellm/integrations/otel/emitter.py

@@ -100,23 +100,19 @@ def _carried_keys(
     return frozenset(key for key, _value in attributes)
 
 
-def attribute_budget(span: Span, reserved: int, overwrites: frozenset[str] = frozenset()) -> int | None:
-    """How many mapped attributes fit on ``span`` next to what it already carries and ``reserved`` more.
+def carried_keys(span: Span) -> frozenset[str]:
+    """The attribute keys ``span`` already holds, empty for a span that exposes none."""
+    if not isinstance(span, ReadableSpan):
+        return frozenset()
+    return _carried_keys(span.attributes or ())
 
-    ``overwrites`` are the mapped keys already present on ``span``: setting one
-    replaces the value in place and consumes no slot against the limit, so only
-    the genuinely new pre-existing keys reduce the budget. Counting the
-    overwritten ones too reserves slots the fit can never spend and sheds
-    indexed message attributes for nothing.
-    """
+
+def attribute_budget(span: Span, reserved: int) -> int | None:
+    """How many new attribute keys fit on ``span`` next to what it already carries and ``reserved`` more."""
     limit: Final = span_attribute_limit(span)
     if limit is None:
         return None
-    if not isinstance(span, ReadableSpan):
-        return limit - reserved
-    existing_keys: Final = _carried_keys(span.attributes or ())
-    fresh: Final = len(existing_keys - overwrites) if overwrites else len(existing_keys)
-    return limit - fresh - reserved
+    return limit - len(carried_keys(span)) - reserved
 
 
 def stamp_error(
@@ -303,8 +299,14 @@ class SpanEmitter:
         )
         stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
         reserved: Final = len(stamped_later.keys() - mapped.keys())
-        for key, value in fit_indexed_messages(mapped, attribute_budget(span, reserved, frozenset(mapped))).items():
-            span.set_attribute(key, value)
+        overwrites: Final = carried_keys(span) & mapped.keys()
+        fitted: Final = fit_indexed_messages(
+            MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
+            attribute_budget(span, reserved),
+        )
+        for key, value in mapped.items():
+            if key in overwrites or key in fitted:
+                span.set_attribute(key, value)
         if error:
             stamped: Final = stamp_error(span, error)
             if stamped is not None and self._event_recorder is not None and role is SpanRole.LLM_CALL:

litellm/integrations/otel/logger.py

@@ -538,8 +538,12 @@ class OpenTelemetryV2(CustomLogger):
         # logger is a success/failure callback only, so ``pre_call`` never reaches it
         # and no carrier exists. The payload plus the request-level provider-handoff
         # stamp (``upstream_started``) is the affirmative signal of a real call; a
-        # gate rejection carries ``is_no_upstream_call`` and gets no span.
-        if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
+        # gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
+        # hit never hands off to a provider, so ``cache_hit`` substitutes for the
+        # upstream stamp, since the payload proves the call completed.
+        if carrier is None and (
+            call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
+        ):
             return None
         try:
             return self._finish_carrier(carrier, call, start_time, end_time)

litellm/integrations/otel/mappers/genai.py

@@ -91,6 +91,8 @@ class GenAIMapper:
         f"{LiteLLM.COST_PREFIX}margin_fixed_amount": lambda d: d.cost.margin_fixed_amount,
         f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent,
         f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount,
+        LiteLLM.CACHE_HIT: lambda d: d.cache_hit,
+        LiteLLM.SAVED_CACHE_COST: lambda d: d.saved_cache_cost,
         LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
         LiteLLM.REQUEST_ROUTE: lambda d: d.request_route,
         LiteLLM.REQUEST_PURPOSE: lambda d: d.request_purpose,

litellm/integrations/otel/model/metadata.py

@@ -249,6 +249,10 @@ class LLMCallEvent:
     # ``InternalCallOrigin`` such as ``autorouter_classifier``), ``None`` for the
     # caller's own provider attempt.
     purpose: str | None
+    # True for a response served from the litellm response cache. A cache hit
+    # never hands off to a provider, so ``upstream_started`` stays False even
+    # though the call completed and is worth a span.
+    cache_hit: bool
     # A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
     # span is renamed from the typed payload at close (``finish_span``); this only
     # needs to be reasonable for a span that never gets closed (a leak).
@@ -273,6 +277,7 @@ class LLMCallEvent:
             upstream_started=kwargs.get("api_call_start_time") is not None,
             upstream_start_seconds=_epoch_seconds(kwargs.get("api_call_start_time")),
             purpose=internal_call_origin(payload, kwargs),
+            cache_hit=bool(payload.get("cache_hit")) if payload else False,  # pyright: ignore[reportUnknownMemberType]  # TypedDict .get on a str key is partially unknown
             provisional_span_name=f"{operation.value} {model}".strip(),
             time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
             trace=trace,

litellm/integrations/otel/model/payloads.py

@@ -441,6 +441,8 @@ class LLMCallSpanData:
     embedding_output: EmbeddingOutput | None = None
     promoted_metadata: Mapping[str, str] = field(default_factory=lambda: _EMPTY_METADATA)
     routing_attributes: Mapping[str, RoutingAttributeValue] = field(default_factory=lambda: MappingProxyType({}))
+    cache_hit: bool | None = None
+    saved_cache_cost: float | None = None
 
     @classmethod
     def from_standard_logging_payload(
@@ -456,6 +458,7 @@ class LLMCallSpanData:
         metadata_keys: tuple[str, ...] = (),
     ) -> LLMCallSpanData:
         params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {})
+        cache_hit: Final = as_bool(payload.get("cache_hit")) is True  # pyright: ignore[reportUnknownMemberType]  # TypedDict .get on a str key is partially unknown
         # The single parse of the request's metadata — the request-vs-provider
         # model split, the response model, api base, and identity all come from
         # here rather than being re-derived from the raw payload dicts.
@@ -490,7 +493,7 @@ class LLMCallSpanData:
             error=_parse_error(payload),
             response_cost=as_float(payload.get("response_cost")),
             cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))),
-            server=ServerInfo.from_api_base(context.api_base),
+            server=None if cache_hit else ServerInfo.from_api_base(context.api_base),
             identity=context.identity,
             promoted_metadata=allowlisted_metadata(context.identity.metadata, metadata_keys),
             is_streaming=as_bool(payload.get("stream")),
@@ -507,6 +510,8 @@ class LLMCallSpanData:
             session_id=session_id or None,
             embedding_output=embedding_output if capture_content else None,
             routing_attributes=routing_decision_attributes(decision),
+            cache_hit=cache_hit,
+            saved_cache_cost=as_float(payload.get("saved_cache_cost")) if cache_hit else None,  # pyright: ignore[reportUnknownMemberType]  # TypedDict .get on a str key is partially unknown
         )
 
 

litellm/integrations/otel/model/semconv.py

@@ -340,6 +340,8 @@ class LiteLLM:
     # semconv key for an MCP server's *name* (the convention uses ``server.address``
     # for its network location), so it lives under the vendor namespace.
     MCP_SERVER_NAME: Final = "litellm.mcp.server.name"
+    CACHE_HIT: Final = "litellm.cache_hit"
+    SAVED_CACHE_COST: Final = "litellm.saved_cache_cost"
 
 
 class Metric:

litellm/integrations/prometheus.py

@@ -50,10 +50,12 @@ from litellm.litellm_core_utils.core_helpers import (
     get_litellm_metadata_from_kwargs,
     get_metadata_variable_name_from_kwargs,
 )
+from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider
 from litellm.litellm_core_utils.service_tier_utils import (
     get_service_tier_from_standard_logging_payload,
 )
 from litellm.llms.github_copilot.per_user_auth import github_copilot_user_session_from
+from litellm.llms.openai_like.json_loader import JSONProviderRegistry
 from litellm.models.end_user import LiteLLM_EndUserTable
 from litellm.models.team import LiteLLM_TeamTable
 from litellm.models.user import LiteLLM_UserTable
@@ -64,6 +66,7 @@ from litellm.repositories.organization_repository import OrganizationRepository
 from litellm.repositories.table_repositories import EndUserRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
+from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.integrations.prometheus import *
 from litellm.types.integrations.prometheus import (
@@ -264,6 +267,29 @@ _POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, F
 _SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)])
 _DEFAULT_SERIES_CLEANUP_INTERVAL_SECONDS: Final = 60.0
 _ADDITIONAL_HEADERS: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
+_LOGGED_FIELDS: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_NO_LOGGED_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+
+
+def _logged_fields(value: object) -> Mapping[str, object]:
+    try:
+        return _LOGGED_FIELDS.validate_python(value)
+    except ValidationError:
+        return _NO_LOGGED_FIELDS
+
+
+def _is_llm_provider(name: str) -> bool:
+    routable: Final = (
+        name in litellm.provider_list or JSONProviderRegistry.exists(name) or is_registered_custom_provider(name)
+    )
+    return routable and name not in litellm._known_custom_logger_compatible_callbacks  # pyright: ignore[reportPrivateUsage]  # the router treats these prefixes as prompt-management deployments
+
+
+def _deployment_model_provider(deployment: object) -> str:
+    if not isinstance(deployment, str):
+        return ""
+    prefix, separator, _ = deployment.partition("/")
+    return prefix if separator and _is_llm_provider(prefix) else ""
 
 
 def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
@@ -339,16 +365,6 @@ def _get_proxy_llm_router() -> Router | None:
     return llm_router
 
 
-_LOGGED_FIELDS: Final = TypeAdapter(Mapping[str, object])
-
-
-def _logged_fields(value: object) -> Mapping[str, object]:
-    try:
-        return _LOGGED_FIELDS.validate_python(value)
-    except ValidationError:
-        return MappingProxyType({})
-
-
 _CALLER_CREDENTIAL_FIELDS: Final = frozenset(
     {
         "aws_access_key_id",
@@ -2944,6 +2960,25 @@ class PrometheusLogger(CustomLogger):
             )
             return None
 
+    @staticmethod
+    def _extract_deployment_api_provider(request_kwargs: Mapping[str, object]) -> str:
+        """
+        Provider of the deployment a failed call was routed to: the resolved
+        ``custom_llm_provider`` on ``litellm_params``, then the one recorded on
+        ``standard_logging_object``, then the provider prefix of the deployment
+        model the router wrote into the request metadata (``azure/my-gpt`` ->
+        ``azure``) when that prefix is an LLM provider the router accepts.
+        Returns an empty string otherwise.
+        """
+        litellm_params: Final = _logged_fields(request_kwargs.get("litellm_params"))
+        provider: Final = litellm_params.get("custom_llm_provider") or _logged_fields(
+            request_kwargs.get("standard_logging_object")
+        ).get("custom_llm_provider")
+        if isinstance(provider, str) and provider:
+            return provider
+        deployment_metadata: Final = find_deployment_metadata(litellm_params)
+        return _deployment_model_provider(deployment_metadata.get("deployment") if deployment_metadata else None)
+
     @staticmethod
     def _extract_api_provider_from_exception(exception: Exception) -> str | None:
         if not isinstance(exception, litellm.exceptions.RateLimitError):
@@ -3222,7 +3257,9 @@ class PrometheusLogger(CustomLogger):
                 label_litellm_model_name = litellm_model_name
                 label_model_id = model_id
                 label_api_base = api_base
-                label_api_provider = llm_provider
+                label_api_provider = self._extract_deployment_api_provider(
+                    _LOGGED_FIELDS.validate_python(request_kwargs)
+                )
                 label_requested_model = model_group or litellm_model_name
                 label_model_group = model_group
             else:

litellm/integrations/websearch_interception/transformation.py

@@ -5,7 +5,7 @@ Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format.
 """
 
 import json
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from typing import Final
 
 from pydantic import TypeAdapter
@@ -539,7 +539,8 @@ class WebSearchTransformation:
             result: SearchResponse from litellm.asearch()
 
         Returns:
-            Formatted text with Title, URL, Snippet for each result
+            Formatted text with Title, URL, Snippet for each result, or the response
+            without its provider_metadata (the caller's billing envelope) when there are no results
         """
         # Convert SearchResponse to string
         if hasattr(result, "results") and result.results:
@@ -548,6 +549,9 @@ class WebSearchTransformation:
                 [f"Title: {r.title}\nURL: {r.url}\nSnippet: {r.snippet}" for r in result.results]
             )
         else:
-            search_result_text = str(result)
+            fields: Final[Mapping[str, object]] = dict(result)
+            search_result_text = " ".join(
+                f"{key}={value!r}" for key, value in fields.items() if key != "provider_metadata"
+            )
 
         return search_result_text

litellm/litellm_core_utils/completion_timeout.py

@@ -2,15 +2,25 @@
 
 from __future__ import annotations
 
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
+from enum import Enum
+from functools import partial
+from types import MappingProxyType
 from typing import Final
 
 import httpx
 from openai import Timeout as SDKTimeout
+from typing_extensions import assert_never
 
 from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS
 
 
+class StreamKind(Enum):
+    NOT_STREAMED = "not_streamed"
+    LIVE = "live"
+    FAKED = "faked"
+
+
 class CompletionTimeout:
     """Resolves HTTP timeout for ``completion()`` from model vs global settings."""
 
@@ -36,17 +46,91 @@ class CompletionTimeout:
             return COMPLETION_HTTP_FALLBACK_SECONDS
         return float(global_timeout)
 
+    @staticmethod
+    def _longer_limit(first: float | None, second: float | None) -> float | None:
+        return None if first is None or second is None else max(first, second)
+
+    @staticmethod
+    def _longer(first: float | httpx.Timeout, second: float | httpx.Timeout) -> float | httpx.Timeout:
+        if not isinstance(first, httpx.Timeout) and not isinstance(second, httpx.Timeout):
+            return max(first, second)
+        first_limits: Final = httpx.Timeout(first)
+        second_limits: Final = httpx.Timeout(second)
+        longer: Final = CompletionTimeout._longer_limit
+        return httpx.Timeout(
+            connect=longer(first_limits.connect, second_limits.connect),
+            read=longer(first_limits.read, second_limits.read),
+            write=longer(first_limits.write, second_limits.write),
+            pool=longer(first_limits.pool, second_limits.pool),
+        )
+
+    @staticmethod
+    def for_stream(
+        stream_kind: StreamKind, request_timeout: float | httpx.Timeout, stream_timeout: float | httpx.Timeout | None
+    ) -> float | httpx.Timeout:
+        """
+        ``stream_timeout`` bounds each read of a live stream, so it replaces the request timeout there. A fake
+        stream is one non-streaming request whose first byte arrives when generation ends, so it takes the longer
+        of each limit and never fails sooner than the same call would with either one alone.
+        """
+        if stream_timeout is None:
+            return request_timeout
+        match stream_kind:
+            case StreamKind.NOT_STREAMED:
+                return request_timeout
+            case StreamKind.LIVE:
+                return stream_timeout
+            case StreamKind.FAKED:
+                return CompletionTimeout._longer(request_timeout, stream_timeout)
+            case _:
+                assert_never(stream_kind)
+
+    @staticmethod
+    def as_limit(timeout: object) -> float | httpx.Timeout:
+        if isinstance(timeout, (httpx.Timeout, SDKTimeout)):
+            return CompletionTimeout.normalize(timeout)
+        if isinstance(timeout, (float, int, str)):
+            return float(timeout)
+        raise TypeError(f"timeout must be a number or an httpx.Timeout, got {type(timeout).__name__}")
+
+    @staticmethod
+    def stream_request_kwargs(non_stream_timeout: object, stream_timeout: object) -> Mapping[str, object]:
+        if stream_timeout is None:
+            return MappingProxyType({"timeout": non_stream_timeout})
+        return MappingProxyType(
+            {
+                "timeout": None
+                if non_stream_timeout is None
+                else CompletionTimeout.for_stream(
+                    StreamKind.FAKED,
+                    CompletionTimeout.as_limit(non_stream_timeout),
+                    CompletionTimeout.as_limit(stream_timeout),
+                ),
+                "stream_timeout": stream_timeout,
+            }
+        )
+
+    @staticmethod
+    def _coerce(
+        timeout: object, custom_llm_provider: str, supports_httpx_timeout: Callable[[str], bool]
+    ) -> float | httpx.Timeout:
+        limit: Final = CompletionTimeout.as_limit(timeout)
+        if not isinstance(limit, httpx.Timeout) or supports_httpx_timeout(custom_llm_provider):
+            return limit
+        return float(limit.read) if limit.read is not None else COMPLETION_HTTP_FALLBACK_SECONDS
+
     @staticmethod
     def resolve(
         model_timeout: float | str | httpx.Timeout | SDKTimeout | None,
-        kwargs: dict,
+        kwargs: Mapping[str, object],
         custom_llm_provider: str,
         *,
         global_timeout: float | str | None,
         supports_httpx_timeout: Callable[[str], bool],
+        stream_kind: StreamKind,
     ) -> float | httpx.Timeout:
         """
-        Resolution order (first non-None wins):
+        Request timeout resolution order (first non-None wins):
 
         1. ``model_timeout`` (call argument / merged ``litellm_params``)
         2. ``kwargs["timeout"]``
@@ -54,24 +138,20 @@ class CompletionTimeout:
         4. ``global_timeout`` (the explicitly-configured ``litellm.request_timeout``),
            or 600 when nothing was configured.
 
+        A stream then applies ``kwargs["stream_timeout"]`` per :meth:`for_stream`.
         Coerce :class:`httpx.Timeout` when the provider does not support it.
         """
-        resolved: float | str | httpx.Timeout | SDKTimeout
-        if model_timeout is not None:
-            resolved = model_timeout
-        elif kwargs.get("timeout") is not None:
-            resolved = kwargs["timeout"]
-        elif kwargs.get("request_timeout") is not None:
-            resolved = kwargs["request_timeout"]
-        else:
-            resolved = CompletionTimeout._fallback_when_no_explicit_timeout(global_timeout)
-
-        if isinstance(resolved, (httpx.Timeout, SDKTimeout)):
-            normalized: Final = CompletionTimeout.normalize(resolved)
-            if supports_httpx_timeout(custom_llm_provider):
-                return normalized
-            read_timeout: Final = normalized.read
-            return (
-                float(read_timeout) if read_timeout is not None else COMPLETION_HTTP_FALLBACK_SECONDS
-            )  # default 10 min timeout
-        return float(resolved)
+        coerce: Final = partial(
+            CompletionTimeout._coerce,
+            custom_llm_provider=custom_llm_provider,
+            supports_httpx_timeout=supports_httpx_timeout,
+        )
+        non_stream_candidates: Final = (model_timeout, kwargs.get("timeout"), kwargs.get("request_timeout"))
+        explicit: Final = next((candidate for candidate in non_stream_candidates if candidate is not None), None)
+        request_timeout: Final = coerce(
+            explicit if explicit is not None else CompletionTimeout._fallback_when_no_explicit_timeout(global_timeout)
+        )
+        stream_timeout: Final = None if stream_kind is StreamKind.NOT_STREAMED else kwargs.get("stream_timeout")
+        return CompletionTimeout.for_stream(
+            stream_kind, request_timeout, None if stream_timeout is None else coerce(stream_timeout)
+        )

litellm/litellm_core_utils/core_helpers.py

@@ -10,14 +10,16 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_c
 import httpx
 from pydantic import TypeAdapter, ValidationError
 
+import litellm
 from litellm._logging import verbose_logger
+from litellm.constants import KUBERNETES_POD_ROUTING_KEY
 from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
 from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    from litellm.types.utils import ModelResponseStream
+    from litellm.types.utils import ModelResponseStream, StandardLoggingKubernetesPodRouting
 
     Span = _Span | Any
 else:
@@ -92,6 +94,10 @@ def normalize_drop_params(value: object) -> bool | None:
         return None
 
 
+def drops_unsupported_params(drop_params: object) -> bool:
+    return litellm.drop_params is True or normalize_drop_params(drop_params) is True
+
+
 def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool:
     normalized: Final = normalize_drop_params(value)
     if normalized is None and value is not None:
@@ -385,6 +391,54 @@ def proxy_stamped_used_client_oauth_token(metadata: object, litellm_params: Mapp
     return metadata.get("used_client_oauth_token") if isinstance(metadata, Mapping) else None
 
 
+def proxy_stamped_kubernetes_pod_routing(
+    metadata: object,
+    litellm_params: Mapping[str, object] | None,
+) -> "StandardLoggingKubernetesPodRouting | None":
+    litellm_metadata: Final = litellm_params.get("litellm_metadata") if litellm_params is not None else None
+    litellm_routing: Final = _standard_logging_kubernetes_pod_routing(litellm_metadata)
+    if litellm_routing is not None:
+        return litellm_routing
+    return _standard_logging_kubernetes_pod_routing(metadata)
+
+
+def _standard_logging_kubernetes_pod_routing(
+    metadata: object,
+) -> "StandardLoggingKubernetesPodRouting | None":
+    if not isinstance(metadata, Mapping):
+        return None
+    metadata_mapping: Final = cast(  # cast-ok: routing metadata is a string-keyed mapping
+        Mapping[str, object], metadata
+    )
+    routing: Final = metadata_mapping.get(KUBERNETES_POD_ROUTING_KEY)
+    if not isinstance(routing, Mapping):
+        return None
+    routing_mapping: Final = cast(  # cast-ok: pod routing records use string keys
+        Mapping[str, object], routing
+    )
+    service_host: Final = routing_mapping.get("service_host")
+    pod_ip: Final = routing_mapping.get("pod_ip")
+    pod_count: Final = routing_mapping.get("pod_count")
+    selection: Final = routing_mapping.get("selection")
+    if (
+        not isinstance(service_host, str)
+        or not isinstance(pod_ip, str)
+        or not isinstance(pod_count, int)
+        or isinstance(pod_count, bool)
+        or pod_count < 1
+    ):
+        return None
+    if selection in ("round_robin", "session_affinity", "session_affinity_retry"):
+        routing_record: Final[StandardLoggingKubernetesPodRouting] = {
+            "service_host": service_host,
+            "pod_ip": pod_ip,
+            "pod_count": pod_count,
+            "selection": selection,
+        }
+        return routing_record
+    return None
+
+
 def get_litellm_metadata_from_kwargs(kwargs: dict):
     """
     Helper to get litellm metadata from all litellm request kwargs

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -1196,6 +1196,9 @@ def _map_sagemaker_exception(
             )
 
 
+_VERTEX_403_STATUS_CODE_PATTERN: Final = re.compile(r"(?<![0-9A-Za-z])403(?![0-9A-Za-z])")
+
+
 def _map_vertex_exception(
     *,
     model: str,
@@ -1206,6 +1209,8 @@ def _map_vertex_exception(
     exception_provider: str,
     extra_information: str,
 ) -> None:
+    status_code: Final[object] = getattr(original_exception, "status_code", None)
+    body_code: Final[int | None] = _get_body_error_code(error_str)
     if "Vertex AI API has not been used in project" in error_str or "Unable to find your project" in error_str:
         raise BadRequestError(
             message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
@@ -1252,7 +1257,11 @@ def _map_vertex_exception(
             llm_provider=custom_llm_provider,
             litellm_debug_info=extra_information,
         )
-    elif "403" in error_str:
+    elif (
+        (not isinstance(status_code, int) or status_code in (403, 500))
+        and body_code in (None, 403)
+        and _VERTEX_403_STATUS_CODE_PATTERN.search(error_str)
+    ):
         raise BadRequestError(
             message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
             model=model,
@@ -1306,7 +1315,7 @@ def _map_vertex_exception(
     elif (
         isinstance(getattr(original_exception, "status_code", None), int)
         and 500 <= original_exception.status_code < 600
-        and _get_body_error_code(error_str) == 429
+        and body_code == 429
     ):
         # upstream gateway wraps a 429 inside a 5xx envelope
         # e.g. HTTP 500/503 with {"error":{"code":429,...}}.
@@ -2413,6 +2422,35 @@ def _map_exception_by_status(
             )
 
 
+_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
+_URL_SCHEME: Final = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*")
+_HOST_WITH_OPTIONAL_PORT: Final = re.compile(
+    r"(?:(?:[A-Za-z0-9-]+\.)+[A-Za-z0-9-]+|localhost|\[[0-9A-Fa-f:.]+\])(?::\d+)?"
+)
+
+
+def with_deployment_api_base(litellm_params: object) -> Mapping[str, object]:
+    params: Final = as_str_mapping(litellm_params) or _NO_LITELLM_PARAMS
+    metadata: Final = as_str_mapping(params.get("metadata"))
+    deployment_api_base: Final = None if metadata is None else metadata.get("api_base")
+    if not isinstance(deployment_api_base, str) or not deployment_api_base:
+        return params
+    return MappingProxyType({**params, "api_base": deployment_api_base})
+
+
+def _api_base_without_credentials(api_base: str) -> str:
+    scheme, separator, _ = api_base.partition("://")
+    scheme_prefix: Final = scheme + separator if separator and _URL_SCHEME.fullmatch(scheme) else ""
+    authority_and_rest: Final = api_base[len(scheme_prefix) :]
+    authority: Final = re.split(r"[/?#]", authority_and_rest, maxsplit=1)[0]
+    after_authority: Final = authority_and_rest[len(authority) :]
+    host: Final = authority.rpartition("@")[2]
+    if "@" in after_authority and _HOST_WITH_OPTIONAL_PORT.fullmatch(host) is None:
+        return ""
+    path: Final = re.split(r"[?#]", after_authority, maxsplit=1)[0]
+    return f"{scheme_prefix}{host}{path}"
+
+
 def _is_guardrail_block(original_exception: Exception) -> bool:
     from litellm.integrations.custom_guardrail import is_guardrail_intervention
 
@@ -2483,8 +2521,9 @@ def exception_type(
                 if isinstance(custom_llm_provider, str) and len(custom_llm_provider) > 0:
                     exception_provider = custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception"
 
-                if _api_base:
-                    extra_information += f"\nAPI Base: `{_api_base}`"
+                shown_api_base: Final = _api_base_without_credentials(_api_base) if _api_base else ""
+                if shown_api_base:
+                    extra_information += f"\nAPI Base: `{shown_api_base}`"
                 if messages and len(messages) > 0 and litellm.redact_messages_in_exceptions is False:
                     extra_information += f"\nMessages: `{messages}`"
 
@@ -2512,7 +2551,8 @@ def exception_type(
             ################################################################################
 
             if (
-                "Request Timeout Error" in error_str
+                isinstance(original_exception, httpx.TimeoutException)
+                or "Request Timeout Error" in error_str
                 or "Request timed out" in error_str
                 or "Timed out generating response" in error_str
                 or "The read operation timed out" in error_str
@@ -2520,7 +2560,7 @@ def exception_type(
                 exception_mapping_worked = True
 
                 raise Timeout(
-                    message=f"APITimeoutError - Request timed out. Error_str: {error_str}",
+                    message=f"APITimeoutError - Request timed out. Error_str: {error_str or type(mappable_exception).__name__}",
                     model=model,
                     llm_provider=custom_llm_provider,
                     litellm_debug_info=extra_information,

litellm/litellm_core_utils/get_litellm_params.py

@@ -142,14 +142,12 @@ def get_base_model_from_litellm_call_metadata(
     if metadata is None:
         return None
     model_info: Final = metadata.get("model_info")
-    if model_info:
-        model_info_mapping: Final = cast(  # cast-ok: model metadata is caller-provided and preserves its mapping shape
-            Mapping[str, object], model_info
-        )
-        return cast(  # cast-ok: model metadata values are caller-provided
-            str | None, model_info_mapping.get("base_model")
-        )
-    return None
+    if not model_info:
+        return None
+    base_model: Final = cast(  # cast-ok: model metadata is caller-provided and preserves its mapping shape
+        Mapping[str, object], model_info
+    ).get("base_model")
+    return base_model if isinstance(base_model, str) else None
 
 
 _get_base_model_from_litellm_call_metadata = get_base_model_from_litellm_call_metadata

litellm/litellm_core_utils/get_model_cost_map.py

@@ -324,10 +324,16 @@ def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _Fetc
     return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
 
 
-def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
+def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable:
+    try:
+        return urllib.parse.urlsplit(url)
+    except ValueError as error:
+        return ModelCostMapReloadUnavailable(reason=f"invalid URL {url}: {error}")
+
+
+def _attempt_fetch_s3(parts: urllib.parse.SplitResult, url: str, timeout: int) -> _FetchAttemptOutcome:
     from botocore.exceptions import BotoCoreError, ClientError
 
-    parts: Final = urllib.parse.urlsplit(url)
     bucket: Final = parts.netloc
     key: Final = parts.path.lstrip("/")
     if not bucket or not key:
@@ -343,9 +349,11 @@ def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
 
 
 async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
-    parts: Final = urllib.parse.urlsplit(url)
+    parts: Final = _split_url(url)
+    if isinstance(parts, ModelCostMapReloadUnavailable):
+        return parts
     if parts.scheme == "s3":
-        return await asyncio.to_thread(_attempt_fetch_s3, url, timeout)
+        return await asyncio.to_thread(_attempt_fetch_s3, parts, url, timeout)
     try:
         response: Final = await client.get(url, timeout=timeout)
     except (httpx.HTTPError, httpx.InvalidURL) as e:
@@ -354,9 +362,11 @@ async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _Fe
 
 
 def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
-    parts: Final = urllib.parse.urlsplit(url)
+    parts: Final = _split_url(url)
+    if isinstance(parts, ModelCostMapReloadUnavailable):
+        return parts
     if parts.scheme == "s3":
-        return _attempt_fetch_s3(url, timeout)
+        return _attempt_fetch_s3(parts, url, timeout)
     try:
         response: Final = client.get(url, timeout=timeout)
     except (httpx.HTTPError, httpx.InvalidURL) as e:
@@ -547,22 +557,39 @@ def get_model_cost_map_loaded_at() -> "datetime | None":
     return _cost_map_source_info.loaded_at
 
 
-def _expand_model_aliases(model_cost: dict) -> dict:
+class AliasedCostEntry(dict[str, object]):
+    __slots__ = ("names",)
+
+    def __init__(self, entry: Mapping[str, object], names: tuple[str, ...]) -> None:
+        super().__init__(entry)
+        self.names: tuple[str, ...] = names
+
+
+def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]:
+    return MappingProxyType(
+        {
+            owner: (owner, *(alias for alias, alias_owner in alias_owners.items() if alias_owner == owner))
+            for owner in dict.fromkeys(alias_owners.values())
+        }
+    )
+
+
+def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]:
     """
     Expand ``aliases`` lists in model cost entries into top-level entries.
 
-    Each alias gets a reference to the **same** dict object as the canonical
-    entry (zero memory overhead).  The ``aliases`` key is removed from the
-    entry so downstream code never sees it.
+    Each alias gets a reference to the **same** ``AliasedCostEntry`` as the
+    canonical entry, which carries every name it is served under.  The
+    ``aliases`` key is removed from the entry so downstream code never sees it.
 
     If an alias collides with an existing canonical entry the alias is
     skipped and a warning is logged.
     """
-    aliases_to_add: Final[dict[str, dict]] = {}
+    alias_owners: Final[dict[str, str]] = {}
     keys_with_aliases: Final[list[str]] = []
 
     for model_name, model_info in model_cost.items():
-        aliases: list | None = model_info.get("aliases")
+        aliases: object = model_info.get("aliases")
         if aliases is None:
             continue
         keys_with_aliases.append(model_name)
@@ -584,7 +611,7 @@ def _expand_model_aliases(model_cost: dict) -> dict:
                     model_name,
                 )
                 continue
-            if alias in aliases_to_add:
+            if alias in alias_owners:
                 verbose_logger.warning(
                     "LiteLLM model alias conflict: alias '%s' (from '%s') "
                     "was already claimed by another entry — skipping.",
@@ -592,13 +619,14 @@ def _expand_model_aliases(model_cost: dict) -> dict:
                     model_name,
                 )
                 continue
-            aliases_to_add[alias] = model_info  # same dict reference
+            alias_owners[alias] = model_name
 
     # Remove the ``aliases`` key from entries so it doesn't pollute model info
     for key in keys_with_aliases:
         model_cost[key].pop("aliases", None)
 
-    model_cost.update(aliases_to_add)
+    for owner, names in _alias_groups(alias_owners).items():
+        model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names)))
     return model_cost
 
 

litellm/litellm_core_utils/get_provider_specific_headers.py

@@ -9,6 +9,7 @@ class ProviderSpecificHeaderUtils:
     def get_provider_specific_headers(
         provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
         custom_llm_provider: str | None,
+        native_anthropic_messages: bool = False,
     ) -> Mapping[str, str]:
         """
         Get the provider specific headers for the given custom llm provider.
@@ -17,7 +18,8 @@ class ProviderSpecificHeaderUtils:
         carries its own comma-separated provider list, so headers that are safe for several
         providers and headers that are safe for exactly one can travel on the same request
         without sharing a scope. Entries whose provider list does not contain
-        `custom_llm_provider` contribute nothing.
+        `custom_llm_provider` contribute nothing, and entries marked
+        `native_anthropic_messages_only` contribute only to a provider's native Messages route.
 
         Returns:
             Dict: The provider specific headers for the given custom llm provider
@@ -31,6 +33,12 @@ class ProviderSpecificHeaderUtils:
 
         matched_headers: Final = {}
         for scoped_header in scoped_headers:
+            if (
+                "native_anthropic_messages_only" in scoped_header
+                and scoped_header["native_anthropic_messages_only"]
+                and not native_anthropic_messages
+            ):
+                continue
             stored_providers = scoped_header.get("custom_llm_provider", "")
             provider_list = [p.strip() for p in stored_providers.split(",")]
             if custom_llm_provider in provider_list:

litellm/litellm_core_utils/litellm_logging.py

@@ -13,12 +13,13 @@ import sys
 import time
 import traceback
 from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from datetime import datetime as dt_object
 from functools import lru_cache
 from types import MappingProxyType, TracebackType
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
 
-from httpx import Response
+from httpx import Response, Timeout
 from pydantic import BaseModel, JsonValue, TypeAdapter
 
 import litellm
@@ -42,6 +43,7 @@ from litellm.constants import (
     DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
     EMPTY_MAPPING,
     INTERNAL_CALLBACK_NAMES,
+    KUBERNETES_POD_ROUTING_KEY,
     PROVIDER_REQUEST_ID_HEADERS,
     REDACTED_BY_LITELLM,
 )
@@ -74,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import (
 from litellm.litellm_core_utils.core_helpers import (
     get_provider_response_headers_from_hidden_params,
     is_expected_client_error,
+    proxy_stamped_kubernetes_pod_routing,
     proxy_stamped_used_client_oauth_token,
     reconstruct_model_name,
     set_response_cost_in_hidden_params,
@@ -148,6 +151,7 @@ from litellm.types.llms.openai import (
 from litellm.types.mcp import MCPPostCallResponseObject
 from litellm.types.prompts.init_prompts import PromptSpec
 from litellm.types.rerank import RerankResponse
+from litellm.types.router import RouterRateLimitError
 from litellm.types.utils import (
     DEPLOYMENT_SCOPED_PRICING_FIELDS,
     CachingDetails,
@@ -297,7 +301,9 @@ else:
     _GENERIC_API_LOGGER_CLS: Final = GenericAPILogger
 _in_memory_loggers: Final[list[CustomLogger]] = []
 
-_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(("used_client_oauth_token", "usage_object"))
+_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(
+    ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY)
+)
 _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = (
     frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS
 )
@@ -558,6 +564,12 @@ def mask_api_base_credentials(api_base: str) -> str:
     return api_base[:key_end] + "*" * 5 + api_base[-4:]
 
 
+@dataclass(frozen=True, slots=True)
+class UncachedPrice:
+    cost: float | None
+    cost_breakdown: CostBreakdown | None
+
+
 def _timestamp_seconds(moment: object) -> float | None:
     if isinstance(moment, datetime.datetime):
         return moment.timestamp()
@@ -627,15 +639,15 @@ class Logging(LiteLLMLoggingBaseClass):
         self,
         model: str,
         messages,
-        stream,
+        stream: bool | None,
         call_type,
         start_time: datetime.datetime,
         litellm_call_id: str,
         function_id: str,
         litellm_trace_id: str | None = None,
         dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = None,
-        dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = None,
-        dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = None,
+        dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
+        dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
         dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
         dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
         applied_guardrails: list[str] | None = None,
@@ -664,6 +676,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.message_copy_duration_ms: float = (time.time() - _copy_start) * 1000
         self.callback_duration_ms: float = 0.0
         self.stream = stream
+        self.applied_timeout: float | Timeout | None = None
         self.start_time = start_time  # log the call start time
         self.call_type = call_type
         self.relays_llm_passthrough: Final = call_type in (
@@ -710,8 +723,10 @@ class Logging(LiteLLMLoggingBaseClass):
 
         # Initialize dynamic callbacks
         self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks
-        self.dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = dynamic_success_callbacks
-        self.dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = (
+        self.dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
+            dynamic_success_callbacks
+        )
+        self.dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
             dynamic_async_success_callbacks
         )
         self.dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = dynamic_failure_callbacks
@@ -778,6 +793,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.defer_async_logging: bool = False
         self.enqueue_deferred_logging: Callable[[], None] | None = None
         self._async_success_scheduled: bool = False
+        self._sync_success_scheduled: bool = False
         self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
@@ -811,6 +827,10 @@ class Logging(LiteLLMLoggingBaseClass):
         new_call.sync_streaming_chunks = []
         return new_call
 
+    def reset_caching_details(self) -> None:
+        """Forget an earlier cache lookup's result, so a call that looks up again reports only its latest."""
+        self.caching_details = None
+
     def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
         """Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
         self.response_timing_metrics = dict(timing_metrics)
@@ -974,6 +994,16 @@ class Logging(LiteLLMLoggingBaseClass):
                 return model_id
         return None
 
+    def _routed_deployment_model(self) -> str | None:
+        litellm_params: Final = self._model_call_details_view.get("litellm_params")
+        metadata: Final = (
+            StandardLoggingPayloadSetup.merge_litellm_metadata(_LITELLM_PARAMS.validate_python(litellm_params))
+            if isinstance(litellm_params, dict)
+            else {}
+        )
+        deployment: Final = metadata.get("deployment")
+        return deployment if isinstance(deployment, str) else None
+
     def get_deployment_model_for_cost(self) -> str | None:
         """The provider-qualified model to price against.
 
@@ -1045,6 +1075,11 @@ class Logging(LiteLLMLoggingBaseClass):
         if "custom_llm_provider" in self.model_call_details:
             self.custom_llm_provider = self.model_call_details["custom_llm_provider"]
 
+    def record_applied_timeout(self, timeout: float | Timeout) -> None:
+        self.applied_timeout = timeout
+        self.litellm_params["timeout"] = timeout
+        self.model_call_details["litellm_params"] = self.litellm_params
+
     def update_from_kwargs(
         self,
         kwargs: dict,
@@ -1890,6 +1925,7 @@ class Logging(LiteLLMLoggingBaseClass):
         cache_hit: bool | None = None,
         litellm_model_name: str | None = None,
         router_model_id: str | None = None,
+        ignore_stamped_cost: bool = False,
     ) -> float | None:
         """
         Calculate response cost using result + logging object variables.
@@ -1925,7 +1961,9 @@ class Logging(LiteLLMLoggingBaseClass):
         ):
             hidden_params: Final = result_hidden_params
             if (
-                "response_cost" in hidden_params and hidden_params["response_cost"] is not None
+                not ignore_stamped_cost
+                and "response_cost" in hidden_params
+                and hidden_params["response_cost"] is not None
             ):  # use cost if already calculated
                 self._record_zero_cost_diagnostic(
                     priced_result,
@@ -2051,6 +2089,20 @@ class Logging(LiteLLMLoggingBaseClass):
 
     _response_cost_calculator = response_cost_calculator
 
+    def uncached_price(self, result: object) -> UncachedPrice:
+        previous_cost_breakdown: Final = self.cost_breakdown
+        previous_billed_token_rates: Final = self.billed_token_rates
+        debug_key: Final = "response_cost_failure_debug_information"
+        had_failure_debug_info: Final = debug_key in self.model_call_details
+        try:
+            cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
+            return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown)
+        finally:
+            self.cost_breakdown = previous_cost_breakdown
+            self.billed_token_rates = previous_billed_token_rates
+            if not had_failure_debug_info:
+                self.model_call_details.pop(debug_key, None)
+
     def _record_zero_cost_diagnostic(
         self,
         result: object,
@@ -2219,6 +2271,7 @@ class Logging(LiteLLMLoggingBaseClass):
             and litellm_params.get(CallTypes.agenerate_content.value, False) is not True
             and litellm_params.get(CallTypes.agenerate_content_stream.value, False) is not True
             and litellm_params.get(CallTypes.arealtime.value, False) is not True
+            and litellm_params.get(CallTypes.aresponses_websocket.value, False) is not True
         )
 
     _is_sync_litellm_request = is_sync_litellm_request
@@ -2284,7 +2337,7 @@ class Logging(LiteLLMLoggingBaseClass):
             **kwargs,
         )
 
-        if not self._should_run_sync_callbacks_for_async_calls():
+        if not self.stream or not self._should_run_sync_callbacks_for_async_calls():
             return
 
         executor.submit(
@@ -2412,7 +2465,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self,
         event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"],
     ) -> None:
-        if self.stream is not None and self.stream is True and event_type in ["async_success", "sync_success"]:
+        if self.stream is True and event_type in ["async_success", "sync_success"]:
             """
             Ignore check on stream, as there can be multiple chunks
             """
@@ -2474,7 +2527,8 @@ class Logging(LiteLLMLoggingBaseClass):
 
         elif self.call_type == CallTypes.aresponses_websocket.value and isinstance(result, list):  # pyright: ignore[reportUnknownMemberType]  # Logging.call_type is untyped
             logging_result = ResponsesWebSocketTokenUsageProcessor.create_logging_object(
-                results=result  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+                results=result,  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+                litellm_model_name=self._routed_deployment_model(),
             )
 
         elif self.relays_llm_passthrough and isinstance(result, Response):
@@ -2909,6 +2963,8 @@ class Logging(LiteLLMLoggingBaseClass):
         discarded as a duplicate of the in-progress create.
         """
         self.model_call_details.pop("has_logged_async_success", None)
+        self.model_call_details.pop("has_logged_sync_success", None)
+        self._sync_success_scheduled = False
         litellm_params = self.model_call_details.get("litellm_params")
         if not isinstance(litellm_params, dict):
             return
@@ -3031,7 +3087,7 @@ class Logging(LiteLLMLoggingBaseClass):
             complete_streaming_response: (
                 ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
             ) = None
-            if "complete_streaming_response" in self.model_call_details:
+            if "complete_streaming_response" in self.model_call_details and not for_async_request:
                 return  # break out of this.
             complete_streaming_response = self._get_assembled_streaming_response(
                 result=result,
@@ -3097,6 +3153,11 @@ class Logging(LiteLLMLoggingBaseClass):
                 dynamic_success_callbacks=self.dynamic_success_callbacks,
                 global_callbacks=litellm.success_callback,
             )
+            delivery_callbacks: Final = (
+                self._get_completed_success_callbacks(asynchronous=False)
+                if for_async_request and not self.stream
+                else callbacks
+            )
 
             ## REDACT MESSAGES ##
             result = redact_message_input_output_from_logging(
@@ -3104,7 +3165,8 @@ class Logging(LiteLLMLoggingBaseClass):
                 result=result,
             )
             ## LOGGING HOOK ##
-            for callback in callbacks:
+            hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
+            for callback in hook_callbacks:
                 if isinstance(callback, CustomGuardrail):
                     from litellm.types.guardrails import GuardrailEventHooks
 
@@ -3134,7 +3196,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
             call_details_view: Final = MappingProxyType[str, object](call_details)
             self.mark_logging_complete(event_type="sync_success")
-            for callback in callbacks:
+            for callback in delivery_callbacks:
                 try:
                     should_run = self.should_run_callback(
                         callback=callback,
@@ -3406,7 +3468,12 @@ class Logging(LiteLLMLoggingBaseClass):
                             )
                     if (
                         isinstance(callback, CustomLogger)
-                        and is_sync_request
+                        and (is_sync_request or (for_async_request and not self.stream))
+                        and not (
+                            for_async_request
+                            and self.stream
+                            and callback in (self.dynamic_async_success_callbacks or ())
+                        )
                         and self.call_type
                         != CallTypes.pass_through.value  # pass-through endpoints call async_log_success_event
                     ):  # custom logger class
@@ -3423,7 +3490,7 @@ class Logging(LiteLLMLoggingBaseClass):
                                 result = call_details_view["complete_response"]
 
                             callback.log_success_event(
-                                kwargs=call_details,
+                                kwargs=callback.redact_standard_logging_payload_from_model_call_details(call_details),
                                 response_obj=result,
                                 start_time=start_time,
                                 end_time=end_time,
@@ -3665,6 +3732,9 @@ class Logging(LiteLLMLoggingBaseClass):
             dynamic_success_callbacks=self.dynamic_async_success_callbacks,
             global_callbacks=litellm._async_success_callback,
         )
+        delivery_callbacks: Final = (
+            self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks
+        )
 
         result = redact_message_input_output_from_logging(
             model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
@@ -3673,7 +3743,8 @@ class Logging(LiteLLMLoggingBaseClass):
 
         ## LOGGING HOOK ##
 
-        for callback in callbacks:
+        hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
+        for callback in hook_callbacks:
             try:
                 if isinstance(callback, CustomGuardrail):
                     from litellm.types.guardrails import GuardrailEventHooks
@@ -3711,7 +3782,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
         self.mark_logging_complete(event_type="async_success")
 
-        for callback in callbacks:
+        for callback in delivery_callbacks:
             # check if callback can run for this request
             litellm_params = self.model_call_details.get("litellm_params", {})
             should_run = self.should_run_callback(
@@ -3846,6 +3917,13 @@ class Logging(LiteLLMLoggingBaseClass):
                     traceback.format_exc(),
                 )
                 self._handle_callback_failure(callback=callback)
+        if not self.stream:
+            try:
+                self.handle_sync_success_callbacks_for_async_calls(
+                    result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit
+                )
+            except RuntimeError:
+                verbose_logger.exception("Could not schedule synchronous success callbacks")
         await flush_post_call_redis_batches()
 
     def _handle_callback_failure(self, callback: object):
@@ -4269,6 +4347,10 @@ class Logging(LiteLLMLoggingBaseClass):
         if self._should_run_sync_callbacks_for_async_calls() is False:
             return
 
+        if not self.stream and self._sync_success_scheduled:
+            return
+        if not self.stream:
+            self._sync_success_scheduled = True
         executor.submit(
             self.success_handler,
             result,
@@ -4287,11 +4369,57 @@ class Logging(LiteLLMLoggingBaseClass):
     ) -> None:
         executor.submit(self.success_handler, result, start_time, end_time, cache_hit)
 
+    def _get_completed_success_callbacks(
+        self, *, asynchronous: bool
+    ) -> tuple[str | Callable[..., object] | CustomLogger, ...]:
+        sync_callbacks: Final = self.get_combined_callback_list(
+            self.dynamic_success_callbacks, litellm.success_callback
+        )
+        async_callbacks: Final = self.get_combined_callback_list(
+            self.dynamic_async_success_callbacks,
+            litellm._async_success_callback,  # pyright: ignore[reportPrivateUsage]  # shared logging registry
+        )
+        legacy_sync_callbacks: Final = self._remove_internal_litellm_callbacks(
+            self._remove_internal_custom_logger_callbacks(sync_callbacks)
+        )
+        selected: Final = async_callbacks if asynchronous else sync_callbacks
+        other: Final = sync_callbacks if asynchronous else async_callbacks
+        return tuple(
+            callback
+            for callback in self.get_combined_callback_list(selected, other)
+            if (
+                (
+                    (
+                        getattr(callback.async_log_success_event, "__func__", callback.async_log_success_event)
+                        is not CustomLogger.async_log_success_event
+                    )
+                    == asynchronous
+                    and (
+                        asynchronous
+                        or getattr(callback.log_success_event, "__func__", callback.log_success_event)
+                        is not CustomLogger.log_success_event
+                    )
+                )
+                if isinstance(callback, CustomLogger)
+                else callback in selected
+                and (
+                    asynchronous
+                    or (
+                        callback not in async_callbacks
+                        and callback in legacy_sync_callbacks
+                        and not (isinstance(callback, str) and callback in INTERNAL_CALLBACK_NAMES)
+                    )
+                )
+            )
+        )
+
     def _should_run_sync_callbacks_for_async_calls(self) -> bool:
         """
         Returns:
             - bool: True if sync callbacks should be run for async calls. eg. `langfuse`, `s3`
         """
+        if not self.stream:
+            return bool(self._get_completed_success_callbacks(asynchronous=False))
         _combined_sync_callbacks: Final = self.get_combined_callback_list(
             dynamic_success_callbacks=self.dynamic_success_callbacks,
             global_callbacks=litellm.success_callback,
@@ -4321,7 +4449,11 @@ class Logging(LiteLLMLoggingBaseClass):
         _filtered_failure_callbacks = self._remove_internal_litellm_callbacks(_filtered_failure_callbacks)
         return len(_filtered_failure_callbacks) > 0
 
-    def get_combined_callback_list(self, dynamic_success_callbacks: list | None, global_callbacks: list) -> list:
+    def get_combined_callback_list(
+        self,
+        dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None,
+        global_callbacks: list[str | Callable[..., object] | CustomLogger],
+    ) -> list[str | Callable[..., object] | CustomLogger]:
         if dynamic_success_callbacks is None:
             return list(global_callbacks)
         return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
@@ -6148,7 +6280,7 @@ class StandardLoggingPayloadSetup:
     @staticmethod
     def get_standard_logging_metadata(
         metadata: Mapping[str, object] | None,
-        litellm_params: dict | None = None,
+        litellm_params: dict[str, object] | None = None,
         prompt_integration: str | None = None,
         applied_guardrails: list[str] | None = None,
         mcp_tool_call_metadata: StandardLoggingMCPToolCall | None = None,
@@ -6188,6 +6320,8 @@ class StandardLoggingPayloadSetup:
                     prompt_integration=prompt_integration,
                 )
 
+        kubernetes_pod_routing: Final = proxy_stamped_kubernetes_pod_routing(metadata, litellm_params)
+
         # Initialize with default values
         clean_metadata = StandardLoggingMetadata(
             user_api_key_hash=None,
@@ -6224,6 +6358,7 @@ class StandardLoggingPayloadSetup:
             user_api_key_auth_metadata=None,
             team_alias=None,
             team_id=None,
+            kubernetes_pod_routing=kubernetes_pod_routing,
             used_client_oauth_token=resolve_used_client_oauth_token(
                 proxy_stamped_used_client_oauth_token(metadata, litellm_params),
                 custom_llm_provider,
@@ -6336,6 +6471,7 @@ class StandardLoggingPayloadSetup:
         custom_llm_provider: str | None,
         init_response_obj: object,
         api_base: str | None = None,
+        router_model_id: str | None = None,
     ) -> StandardLoggingModelInformation:
         model_cost_name: Final = select_model_name_for_cost_calc(
             model=base_model if custom_pricing else None,
@@ -6343,12 +6479,23 @@ class StandardLoggingPayloadSetup:
             base_model=base_model,
             custom_pricing=custom_pricing,
         )
+        pricing_model_name: Final = (
+            select_model_name_for_cost_calc(
+                model=base_model if custom_pricing else None,
+                completion_response=init_response_obj,
+                base_model=base_model,
+                custom_pricing=custom_pricing,
+                router_model_id=router_model_id,
+            )
+            if router_model_id is not None
+            else model_cost_name
+        )
         if model_cost_name is None:
             model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None)
         else:
             try:
                 _model_cost_information: Final = litellm.get_model_info(
-                    model=model_cost_name,
+                    model=pricing_model_name or model_cost_name,
                     custom_llm_provider=custom_llm_provider,
                     api_base=api_base,
                 )
@@ -6558,6 +6705,7 @@ class StandardLoggingPayloadSetup:
         rate_limit_category: Final = validate_rate_limit_category(getattr(original_exception, "category", None))
         rate_limit_type: Final = validate_rate_limit_type(getattr(original_exception, "rate_limit_type", None))
         budget_error: Final = original_exception if isinstance(original_exception, BudgetExceededError) else None
+        router_error: Final = original_exception if isinstance(original_exception, RouterRateLimitError) else None
         provider_request_id: Final = _get_provider_request_id(original_exception) if original_exception else None
 
         return StandardLoggingPayloadErrorInformation(
@@ -6573,6 +6721,7 @@ class StandardLoggingPayloadSetup:
             error_budget_entity_id=budget_error.entity_id if budget_error else None,
             error_budget_limit=budget_error.max_budget if budget_error else None,
             error_budget_spend=budget_error.current_cost if budget_error else None,
+            error_cooldown_deployments=router_error.cooldown_deployments if router_error else None,
             normalized_error=normalize_error(original_exception, error_status, error_message),
         )
 
@@ -6909,6 +7058,36 @@ def _autorouter_savings_for_payload(
         return None
 
 
+_CACHE_HIT_ZEROED_AMOUNTS: Final = MappingProxyType(
+    {
+        "cache_read_cost": CostBreakdown(cache_read_cost=0.0),
+        "cache_creation_cost": CostBreakdown(cache_creation_cost=0.0),
+        "reasoning_cost": CostBreakdown(reasoning_cost=0.0),
+        "original_cost": CostBreakdown(original_cost=0.0),
+        "discount_amount": CostBreakdown(discount_amount=0.0),
+        "margin_fixed_amount": CostBreakdown(margin_fixed_amount=0.0),
+        "margin_total_amount": CostBreakdown(margin_total_amount=0.0),
+    }
+)
+
+
+def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
+    """Every amount at 0, keeping the pricing basis and rates a fresh call would have had, which savings reports read."""
+    zero_costs: Final = CostBreakdown(input_cost=0.0, output_cost=0.0, total_cost=0.0, tool_usage_cost=0.0)
+    if served is None:
+        return zero_costs
+    zeroed_additional_costs: Final = (
+        CostBreakdown(additional_costs=dict.fromkeys(served["additional_costs"], 0.0))
+        if "additional_costs" in served
+        else CostBreakdown()
+    )
+    return functools.reduce(
+        lambda breakdown, zeroed: breakdown | zeroed,
+        (_CACHE_HIT_ZEROED_AMOUNTS[key] for key in served if key in _CACHE_HIT_ZEROED_AMOUNTS),
+        served | zero_costs | zeroed_additional_costs,
+    )
+
+
 def get_standard_logging_object_payload(
     kwargs: dict | None,
     init_response_obj: object,
@@ -6992,17 +7171,10 @@ def get_standard_logging_object_payload(
             "user", None
         )  # maintain backwards compatibility with old request body check
 
-        saved_cache_cost: float = 0.0
         if cache_hit is True or served_from_cache(init_response_obj):
             id = f"{id}_cache_hit{time.time()}"  # do not duplicate the request id
-        if cache_hit is True:
-            saved_cache_cost = (
-                logging_obj.response_cost_calculator(
-                    result=init_response_obj,
-                    cache_hit=False,
-                )
-                or 0.0
-            )
+        uncached_price: Final = logging_obj.uncached_price(result=init_response_obj) if cache_hit is True else None
+        saved_cache_cost: Final = (uncached_price.cost if uncached_price is not None else None) or 0.0
 
         ## Get model cost information ##
         base_model = get_base_model_from_metadata(model_call_details=kwargs)
@@ -7019,7 +7191,7 @@ def get_standard_logging_object_payload(
 
         # clean up litellm hidden params
         clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
-        if clean_hidden_params["response_cost"] is None and raw_response_cost is not None:
+        if cache_hit is True or (clean_hidden_params["response_cost"] is None and raw_response_cost is not None):
             clean_hidden_params["response_cost"] = llm_response_cost
         if clean_hidden_params["litellm_overhead_time_ms"] is None and status == "success":
             # /v1/messages dict results and the bridge stream wrappers keep it on the logging object;
@@ -7033,6 +7205,7 @@ def get_standard_logging_object_payload(
             custom_llm_provider=kwargs.get("custom_llm_provider"),
             init_response_obj=init_response_obj,
             api_base=litellm_params.get("api_base"),
+            router_model_id=logging_obj.get_router_model_id(),
         )
 
         error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
@@ -7081,7 +7254,14 @@ def get_standard_logging_object_payload(
         ):
             model_name = response_model_name
 
-        request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
+        request_cost_breakdown: Final = cost_breakdown_with_guardrail(
+            (
+                _cache_hit_cost_breakdown(uncached_price.cost_breakdown)
+                if uncached_price is not None
+                else logging_obj.cost_breakdown
+            ),
+            guardrail_cost,
+        )
         captured_baseline: Final = logging_obj.baseline_observation
         autorouter_savings: Final = (
             None
@@ -7261,6 +7441,7 @@ def get_standard_logging_metadata(
         user_api_key_auth_metadata=None,
         team_alias=None,
         team_id=None,
+        kubernetes_pod_routing=None,
         used_client_oauth_token=None,
     )
     if isinstance(metadata, dict):
@@ -7334,6 +7515,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
         requester_ip_address="127.0.0.1",
         requester_metadata=None,
         user_api_key_end_user_id="test_end_user",
+        kubernetes_pod_routing=None,
     )
 
     hidden_params: Final = StandardLoggingHiddenParams(

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1088,6 +1088,25 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
     )
 
 
+def _modality_token_cost(
+    model_info: ModelInfo,
+    modality_cost_key: str,
+    tokens: int,
+    prompt_base_cost: float,
+    service_tier: str | None,
+) -> float:
+    """
+    Audio, image and video prompt tokens bill at their modality rate when the model has one.
+    Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which
+    `_get_token_base_cost` has already resolved for the service tier and long-context tier.
+    """
+    tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier)
+    cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key
+    if model_info.get(cost_key) is None:
+        return float(tokens) * prompt_base_cost
+    return calculate_cost_component(model_info, cost_key, tokens)
+
+
 def _calculate_input_cost(
     prompt_tokens_details: PromptTokensDetailsResult,
     model_info: ModelInfo,
@@ -1118,28 +1137,37 @@ def _calculate_input_cost(
     if prompt_tokens_details["audio_tokens"] and not (
         prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
     ):
-        audio_cost_key: Final = get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
-        prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
+        prompt_cost += _modality_token_cost(
+            model_info,
+            "input_cost_per_audio_token",
+            prompt_tokens_details["audio_tokens"],
+            prompt_base_cost,
+            service_tier,
+        )
 
     ### IMAGE TOKEN COST
     if prompt_tokens_details["image_tokens"] and not (
         prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
     ):
-        # For image token costs:
-        # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
-        image_token_cost_key = "input_cost_per_image_token"
-        if model_info.get(image_token_cost_key) is None:
-            image_token_cost_key = "input_cost_per_token"
-        prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
+        prompt_cost += _modality_token_cost(
+            model_info,
+            "input_cost_per_image_token",
+            prompt_tokens_details["image_tokens"],
+            prompt_base_cost,
+            service_tier,
+        )
 
     ### VIDEO TOKEN COST
     if prompt_tokens_details["video_tokens"] and not (
         prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
     ):
-        video_token_cost_key = "input_cost_per_video_token"
-        if model_info.get(video_token_cost_key) is None:
-            video_token_cost_key = "input_cost_per_token"
-        prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"])
+        prompt_cost += _modality_token_cost(
+            model_info,
+            "input_cost_per_video_token",
+            prompt_tokens_details["video_tokens"],
+            prompt_base_cost,
+            service_tier,
+        )
 
     ### CACHE WRITING COST - Now uses tiered pricing
     if (

litellm/litellm_core_utils/prompt_templates/chat_template_tools.py

@@ -0,0 +1,37 @@
+import json
+import re
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Final
+
+
+@dataclass(frozen=True, slots=True)
+class ChatTemplateToolSection:
+    preamble_tokens: int
+    implicit_system_turn_tokens: int
+    render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
+
+
+def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
+    return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions)
+
+
+def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
+    return "".join(
+        "\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions
+    )
+
+
+_TOOL_SECTIONS: Final = (
+    (
+        re.compile(r"deepseek-v4(?![.\d]|p\d)", re.IGNORECASE),
+        ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas),
+    ),
+    (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
+)
+
+
+def chat_template_tool_section(model: str | None) -> ChatTemplateToolSection | None:
+    if not model:
+        return None
+    return next((section for pattern, section in _TOOL_SECTIONS if pattern.search(model)), None)