Merge origin/main into litellm_json_logs_startup_plaintext

Resolves the litellm/proxy/proxy_server.py import conflict: main split the
litellm._logging import over several lines and this branch added
print_startup_message to it; the merged import keeps main's layout with both.

.githooks/pre-push

@@ -1,90 +0,0 @@
-#!/usr/bin/env bash
-#
-# pre-push — enforce Conventional Branches
-#   https://conventional-branch.github.io/
-#
-# Branch format:  <type>/<description>
-#   <type> must be one of: feature, bugfix, hotfix, release, chore
-#
-# Protected branches (always allowed):
-#   - main
-#   - dependabot/*
-#   - gh-readonly-queue/*
-#
-# Tag pushes and branch deletions are skipped.
-# Bypass: git push --no-verify.
-
-set -eu
-
-ZERO_OID="0000000000000000000000000000000000000000"
-ZERO_OID_SHA256="0000000000000000000000000000000000000000000000000000000000000000"
-ALLOWED_TYPES="feature|bugfix|hotfix|release|chore"
-BRANCH_PATTERN="^(${ALLOWED_TYPES})/.+"
-
-PROTECTED_NAMES="main"
-PROTECTED_PREFIXES="dependabot/ gh-readonly-queue/"
-
-is_protected() {
-    branch="$1"
-    for name in $PROTECTED_NAMES; do
-        if [ "$branch" = "$name" ]; then
-            return 0
-        fi
-    done
-    for prefix in $PROTECTED_PREFIXES; do
-        case "$branch" in "$prefix"*) return 0 ;; esac
-    done
-    return 1
-}
-
-invalid=""
-
-while read -r local_ref local_oid remote_ref remote_oid; do
-    # Branch deletion (no local commit being pushed).
-    if [ "$local_oid" = "$ZERO_OID" ] || [ "$local_oid" = "$ZERO_OID_SHA256" ]; then
-        continue
-    fi
-
-    # Only validate branch pushes; ignore tags and other ref namespaces.
-    case "$remote_ref" in
-        refs/heads/*) ;;
-        *) continue ;;
-    esac
-
-    branch="${remote_ref#refs/heads/}"
-
-    if is_protected "$branch"; then
-        continue
-    fi
-
-    if ! printf '%s' "$branch" | grep -Eq "$BRANCH_PATTERN"; then
-        invalid="$invalid $branch"
-    fi
-done
-
-if [ -n "$invalid" ]; then
-    cat >&2 <<EOF
-✗ Branch name does not follow Conventional Branches.
-
-  Invalid:$invalid
-
-  Expected: <type>/<description>
-
-  Allowed types: feature, bugfix, hotfix, release, chore
-  Examples:
-    feature/weighted-round-robin
-    bugfix/streaming-empty-chunks
-    chore/bump-deps
-    hotfix/auth-bypass
-
-  Protected (always allowed): main, dependabot/*, gh-readonly-queue/*.
-
-See https://conventional-branch.github.io/
-
-Rename with: git branch -m <new-name>
-To bypass (use sparingly): git push --no-verify
-EOF
-    exit 1
-fi
-
-exit 0

.github/CODEOWNERS

@@ -1,2 +1,4 @@
 /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
+/helm/ @yassin-berriai
+/terraform/ @yassin-berriai

.github/e2e-stack/assert_tests_ran.py

@@ -1,69 +0,0 @@
-import os
-import re
-import sys
-import xml.etree.ElementTree as ET
-from pathlib import Path
-from typing import Final
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests/e2e"))
-from coverage_registry.management_cases import MANAGEMENT_CASES
-
-
-def main() -> int:
-    selected: Final = tuple(sys.argv[2:])
-    try:
-        report: Final = ET.parse(Path(sys.argv[1])).getroot()
-    except (ET.ParseError, OSError):
-        _ = sys.stdout.write("::error::could not read the test execution report\n")
-        return 1
-    cases: Final = tuple(report.iter("testcase"))
-    expected_count: Final = os.environ.get("E2E_REQUIRED_TEST_COUNT")
-    passed: Final = frozenset(
-        case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
-    )
-    missing: Final = tuple(path for path in selected if path not in passed)
-    required_nodes: Final = frozenset(case.node for case in MANAGEMENT_CASES if case.node.split("::", 1)[0] in selected)
-    passed_nodes: Final = frozenset(
-        prop.get("value")
-        for case in cases
-        if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
-        for prop in case.findall("./properties/property")
-        if prop.get("name") == "management_node"
-    )
-    missing_nodes: Final = required_nodes - passed_nodes
-    for node in sorted(missing_nodes):
-        _ = sys.stdout.write(f"::error::required management case did not pass: {node}\n")
-    for path in selected:
-        collected: Final = sum(case.get("file") == path for case in cases)
-        skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
-        _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
-        for case in cases:
-            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error", "skipped")):
-                continue
-            outcome = "skipped" if case.find("skipped") is not None else "failed"
-            _ = sys.stdout.write(f"  {outcome}: {case.get('classname', '')}::{case.get('name', '')}\n")
-            for prop in case.findall("./properties/property"):
-                name = prop.get("name", "")
-                value = prop.get("value", "")
-                if name in ("oauth_failure_phase", "oauth_exception_type", "oauth_frame") and re.fullmatch(
-                    r"[A-Za-z0-9_.:<>-]{1,240}", value
-                ):
-                    _ = sys.stdout.write(f"    {name}: {value}\n")
-    if expected_count is not None and (
-        len(cases) != int(expected_count) or any(case.find("skipped") is not None for case in cases)
-    ):
-        _ = sys.stdout.write("::error::required test count was not met or a required case was skipped\n")
-        return 1
-    if (
-        selected
-        and not missing
-        and not missing_nodes
-        and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
-    ):
-        return 0
-    _ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
-    return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/workflows/auto_update_price_and_context_window.yml

@@ -24,12 +24,9 @@ jobs:
       - name: Update JSON Data
         run: |
           uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
-      - name: Regenerate JSON Schema
-        run: |
-          uv run --frozen python ci_cd/generate_model_prices_schema.py
       - name: Create Pull Request
         run: |
-          git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
+          git add model_prices_and_context_window.json
           git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" 
           gh pr create --title "Update model_prices_and_context_window.json file" \
             --body "Automated update for model_prices_and_context_window.json" \

.github/workflows/sync-together-ai-models.yml

@@ -41,10 +41,6 @@ jobs:
           uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
         env:
           TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
-      - name: Regenerate the JSON schema
-        if: steps.existing.outputs.open_pr == ''
-        run: |
-          uv run --frozen python ci_cd/generate_model_prices_schema.py
       - name: Create a pull request when the registry changed
         if: steps.existing.outputs.open_pr == ''
         run: |
@@ -57,8 +53,7 @@ jobs:
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git checkout -b "$branch"
           git add model_prices_and_context_window.json \
-            litellm/model_prices_and_context_window_backup.json \
+            litellm/model_prices_and_context_window_backup.json
-            model_prices_and_context_window.schema.json
           git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
           gh auth setup-git
           git push origin "$branch"

.github/workflows/test-linting.yml

@@ -539,8 +539,8 @@ jobs:
       - name: test_unit_passed_gate
         run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py
 
-      - name: test_e2e_changed_gate
+      - name: test_e2e_idp_stack
-        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_idp_stack.py
 
       - name: test_e2e_metadata
         env:

.github/workflows/test-mcp-oauth-e2e.yml

@@ -1,180 +0,0 @@
-name: MCP OAuth happy path
-
-on:
-  pull_request:
-    paths:
-      - '.github/workflows/test-mcp-oauth-e2e.yml'
-      - '.github/e2e-stack/**'
-      - 'tests/e2e/*.py'
-      - 'tests/e2e/pytest.ini'
-      - 'tests/e2e/idp_realm.json'
-      - 'tests/e2e/mcp/**'
-      - 'litellm/experimental_mcp_client/**'
-      - 'litellm/proxy/_experimental/mcp_server/**'
-      - 'litellm/proxy/auth/**'
-      - 'litellm/proxy/management_endpoints/mcp_management_endpoints.py'
-      - 'litellm/proxy/_types.py'
-      - 'litellm/types/mcp_server/mcp_server_manager.py'
-      - 'litellm/proxy/management_endpoints/*sso*.py'
-      - 'litellm/proxy/management_endpoints/sso/**'
-      - 'litellm/proxy/common_utils/encrypt_decrypt_utils.py'
-      - 'litellm/proxy/proxy_server.py'
-      - 'litellm/proxy/schema.prisma'
-      - 'ui/litellm-dashboard/src/app/connect/**'
-      - 'ui/litellm-dashboard/src/app/mcp/oauth/**'
-      - 'pyproject.toml'
-      - 'uv.lock'
-  workflow_dispatch:
-
-permissions: {}
-
-concurrency:
-  group: mcp-oauth-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  oauth:
-    if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
-    runs-on: ubuntu-latest
-    environment: e2e-changed
-    timeout-minutes: 45
-    permissions:
-      contents: read
-      id-token: write
-    services:
-      postgres:
-        image: postgres:16.6
-        env:
-          POSTGRES_USER: litellm
-          POSTGRES_PASSWORD: dbpassword9090
-          POSTGRES_DB: litellm
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd "pg_isready -U litellm"
-          --health-interval 5s
-          --health-timeout 5s
-          --health-retries 10
-    env:
-      DATABASE_HOST: 127.0.0.1
-      DATABASE_PORT: '5432'
-      DATABASE_USER: litellm
-      DATABASE_PASSWORD: dbpassword9090
-      DATABASE_NAME: litellm
-      DATABASE_URL: postgresql://litellm:dbpassword9090@127.0.0.1:5432/litellm
-      E2E_KEYCLOAK_URL: http://127.0.0.1:8081
-      E2E_KEYCLOAK_ADMIN_USER: admin
-      E2E_KEYCLOAK_ADMIN_PASSWORD: e2e-ephemeral-idp-not-a-secret
-      E2E_FIXTURE_MODE: live
-      E2E_PROVIDER_CACHE: '0'
-      E2E_MCP_OAUTH_LIVE: '1'
-      E2E_REQUIRED_TEST_COUNT: '4'
-    steps:
-      - name: Checkout the tested source
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.sha }}
-          persist-credentials: false
-
-      - name: Require and materialize the upstream login
-        env:
-          STORAGE_STATE: ${{ secrets.E2E_LINEAR_STORAGE_STATE_B64 }}
-        run: |
-          umask 077
-          python3 - <<'PY'
-          import base64
-          import json
-          import os
-          import secrets
-          from pathlib import Path
-          encoded = os.environ.get("STORAGE_STATE", "")
-          if not encoded:
-              raise SystemExit("E2E_LINEAR_STORAGE_STATE_B64 is required; capture and provision a test-account login")
-          state = json.loads(base64.b64decode(encoded, validate=True))
-          if not isinstance(state, dict) or not state.get("cookies"):
-              raise SystemExit("The captured login must contain browser cookies")
-          directory = Path(os.environ["RUNNER_TEMP"]) / "mcp-oauth-private"
-          directory.mkdir(mode=0o700)
-          path = directory / "linear-state.json"
-          path.write_text(json.dumps(state))
-          with open(os.environ["GITHUB_ENV"], "a") as output:
-              output.write(f"E2E_LINEAR_STORAGE_STATE={path}\n")
-              for name in ("LITELLM_MASTER_KEY", "LITELLM_SALT_KEY"):
-                  value = "sk-e2e-" + secrets.token_hex(24)
-                  print(f"::add-mask::{value}")
-                  output.write(f"{name}={value}\n")
-          PY
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: '3.13'
-      - uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: '0.10.9'
-      - uses: ./.github/actions/cache-cargo-build
-      - name: Install the frozen E2E environment
-        run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra proxy-runtime --extra extra_proxy --group ci --group proxy-dev --group e2e-dev
-          uv run --no-sync python scripts/prisma_generate_if_needed.py
-          uv run --no-sync playwright install --with-deps chromium
-
-      - name: Configure license access
-        id: aws
-        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
-        with:
-          role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
-          aws-region: us-east-1
-          role-session-name: mcp-oauth-${{ github.run_id }}
-          role-duration-seconds: 900
-          output-env-credentials: false
-          output-credentials: true
-      - name: Load the E2E license
-        env:
-          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
-          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
-          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
-          AWS_DEFAULT_REGION: us-east-1
-        run: |
-          license="$(aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license --query SecretString --output text)"
-          test -n "${license}"
-          echo "::add-mask::${license}"
-          echo "LITELLM_LICENSE=${license}" >> "${GITHUB_ENV}"
-
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
-        with:
-          node-version-file: ui/litellm-dashboard/.nvmrc
-      - name: Build the gateway consent UI at the tested commit
-        run: |
-          cd ui/litellm-dashboard
-          ../../scripts/with_dashboard_node.sh npm ci
-          ../../scripts/with_dashboard_node.sh npm run build
-          mkdir -p ../../litellm/proxy/_experimental/out
-          cp -r out/. ../../litellm/proxy/_experimental/out/
-          find ../../litellm/proxy/_experimental/out -name '*.html' ! -name index.html | while read -r page; do
-            mkdir -p "${page%.html}"
-            mv "${page}" "${page%.html}/index.html"
-          done
-
-      - name: Prepare the isolated database and IdP
-        run: |
-          umask 077
-          bash .github/e2e-stack/start-idp.sh
-          uv run --no-sync python migrations/run.py > "${RUNNER_TEMP}/mcp-oauth-private/migrations.log" 2>&1
-
-      - name: Run every required OAuth variant without retries
-        run: |
-          umask 077
-          uv run --no-sync pytest -c tests/e2e/pytest.ini tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py \
-            --rootdir=. --reruns 0 --tb=short -o junit_family=xunit1 \
-            --junitxml="${RUNNER_TEMP}/mcp-oauth-private/results.xml" \
-            > "${RUNNER_TEMP}/mcp-oauth-private/pytest.log" 2>&1
-      - name: Report JUnit results and reject skipped or missing cases
-        if: always()
-        run: |
-          uv run --no-sync python .github/e2e-stack/assert_tests_ran.py \
-            "${RUNNER_TEMP}/mcp-oauth-private/results.xml" tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py
-      - name: Remove private login and logs
-        if: always()
-        run: |
-          docker rm -f e2e-keycloak >/dev/null 2>&1 || true
-          rm -rf "${RUNNER_TEMP}/mcp-oauth-private"

.github/workflows/test-unit.yml

@@ -266,6 +266,7 @@ jobs:
               tests/unit/llms/featherless_ai
               tests/unit/llms/firecrawl
               tests/unit/llms/fireworks_ai
+              tests/unit/llms/friendliai
               tests/unit/llms/gdc
               tests/unit/llms/gemini
               tests/unit/llms/gigachat
@@ -410,6 +411,7 @@ jobs:
               tests/unit/embeddings
               tests/unit/endpoints
               tests/unit/files
+              tests/unit/fine_tuning
               tests/unit/images
               tests/unit/interactions
               tests/unit/messages
@@ -658,6 +660,7 @@ jobs:
           - shard: proxy-extras
             test-path: >-
               tests/unit/litellm_proxy_extras
+              tests/unit/db_scripts
             python-version: "3.12"
             workers: 2
             reruns: 0

CONTRIBUTING.md

@@ -39,12 +39,12 @@ git clone https://github.com/YOUR_USERNAME/litellm.git
 cd litellm
 
 # Create a new branch for your feature (see "Commit and Branch Conventions" below)
-git checkout -b feature/your-feature
+git checkout -b litellm_your_feature
 
 # Install development dependencies
 make install-dev
 
-# Install git hooks that enforce commit + branch conventions (one-time, opt-in)
+# Install git hooks that enforce commit conventions (one-time, opt-in)
 make install-hooks
 
 # Verify your setup works
@@ -55,7 +55,7 @@ That's it! Your local development environment is ready.
 
 ## Commit and Branch Conventions
 
-Commits follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) and branches follow [Conventional Branches](https://conventional-branch.github.io/). Run `make install-hooks` once per clone to enable the local git hooks that enforce these — see the [contributor docs](https://docs.litellm.ai/docs/extras/contributing_code#commit-and-branch-conventions) for the full type list, examples, the protected-branch bypass list, and how to opt out.
+Commits follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). Internal contributors name branches with the `litellm_` prefix and no `/`, and fork branches can use any name. Run `make install-hooks` once per clone to enable the local hook that enforces the commit convention. See the [contributor docs](https://docs.litellm.ai/docs/extras/contributing_code#commit-and-branch-conventions) for details
 
 ### 2. Development Workflow
 
@@ -78,8 +78,8 @@ uv run pytest tests/unit/<your_test_file>.py -v
 git add .
 git commit -m "feat(scope): your descriptive commit message"
 
-# Push and create a PR (branch must follow Conventional Branches — see above)
+# Push and create a PR
-git push origin feature/your-feature
+git push origin litellm_your_feature
 ```
 
 ## Adding Testing

Makefile

@@ -22,7 +22,7 @@ help:
 	@echo "  make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
 	@echo "  make install-test-deps  - Install the full local test environment"
 	@echo "  make install-helm-unittest - Install helm unittest plugin"
-	@echo "  make install-hooks      - Install git hooks (Conventional Commits + Branches)"
+	@echo "  make install-hooks      - Install git hooks (Conventional Commits)"
 	@echo "  make check              - Run CI-equivalent lint on staged files, or on the diff vs the base branch when nothing is staged"
 	@echo "  make pre-commit         - Legacy alias for make check"
 	@echo "  make format             - Apply ruff format code formatting"
@@ -120,7 +120,7 @@ install-helm-unittest:
 		helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.8.2; \
 	}
 
-# Install git hooks that enforce Conventional Commits and Conventional Branches.
+# Install git hooks that enforce Conventional Commits.
 # Opt-in: not chained into install-dev.
 install-hooks:
 	./scripts/install_git_hooks.sh

README.md

@@ -384,6 +384,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |
 | [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Nace (`nace`)](https://console.nace.ai/docs/drex) |  |  |  |  |  |  |  |  |  |  |
 | [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
 | [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

backend/Dockerfile

@@ -4,6 +4,33 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
 
 FROM $UV_IMAGE AS uvbin
 
+# ---------- Rust extension ----------
+# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
+FROM $LITELLM_BUILD_IMAGE AS rust-native
+
+WORKDIR /app
+USER root
+
+COPY --from=uvbin /uv /uvx /usr/local/bin/
+
+RUN for i in 1 2 3; do \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
+      [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
+      sleep 5; \
+    done
+
+ENV UV_PYTHON_DOWNLOADS=0
+
+COPY pyproject.toml README.md rust-toolchain.toml ./
+COPY .cargo ./.cargo
+COPY litellm-rust ./litellm-rust
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    mkdir -p litellm/rust_bridge /wheels && \
+    touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
+    uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
+        python -c "import maturin; maturin.build_wheel('/wheels')"
+
 # ---------- Builder ----------
 FROM $LITELLM_BUILD_IMAGE AS builder
 
@@ -54,7 +81,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 COPY . .
 
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv sync --frozen --no-default-groups --no-editable \
+    uv sync --frozen --no-install-project --no-default-groups --no-editable \
         --extra proxy \
         --extra proxy-runtime \
         --group admin-mcp \
@@ -63,6 +90,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra saml \
         --python python3.13
 
+RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
+    uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
+
 RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

backend/routes/allowlist.py

@@ -165,6 +165,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
 BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
     {
         "/admin",
+        "/metrics",
         "/swagger",  # API documentation static assets belong to the backend
         "/mcp",  # lazily-mounted MCP sub-app serves on the backend component
     }

ci_cd/cost_map_guard.py

@@ -1,11 +1,11 @@
 """Guard the cost map on pull requests.
 
-Every pull request whose diff against its merge base touches one of the three cost map files gets the file
+Every pull request whose diff against its merge base touches one of the two cost map files gets the file
-checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
+checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
-map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
+skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
-and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
+branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
-(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
+get the file checks and additionally may only touch those two files or the legacy schema
-files and may only add or update models.
+path and may only add or update models.
 """
 
 from __future__ import annotations
@@ -19,13 +19,12 @@ from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final
 
-from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
-
 COST_MAP_PATH: Final = "model_prices_and_context_window.json"
 BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
-SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
+LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
-GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
+GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
 BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
 EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
 
 CostMap = dict[str, object]
@@ -35,7 +34,6 @@ CostMap = dict[str, object]
 class Snapshot:
     cost_map: str
     backup: str
-    schema: str
 
 
 def _parse_object(text: str, path: str) -> CostMap | str:
@@ -46,13 +44,6 @@ def _parse_object(text: str, path: str) -> CostMap | str:
     return parsed if isinstance(parsed, dict) else f"{path} must be a JSON object at the root"
 
 
-def _rendered_schema(cost_map: CostMap) -> str:
-    try:
-        return render(build_schema(cost_map))
-    except SystemExit as error:
-        return str(error)
-
-
 def _evaluation_output_price_failure(
     key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
 ) -> str | None:
@@ -78,29 +69,13 @@ def _file_failures(
     *,
     evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
 ) -> tuple[str, ...]:
-    schema_text: Final = _rendered_schema(head_map)
-    if not schema_text.startswith("{"):
-        return (schema_text,)
     backup_failure: Final = (
         ()
         if head.backup == head.cost_map
         else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",)
     )
-    schema_failure: Final = (
-        ()
-        if head.schema == schema_text
-        else (
-            f"{SCHEMA_PATH} is out of sync with {COST_MAP_PATH}; "
-            "run `python ci_cd/generate_model_prices_schema.py` and commit the result",
-        )
-    )
     return (
         *backup_failure,
-        *schema_failure,
-        *(
-            f"{COST_MAP_PATH} does not validate against its schema: {error}"
-            for error in validation_errors(head_map, json.loads(schema_text))[:20]
-        ),
         *(
             failure
             for key, entry in head_map.items()
@@ -135,7 +110,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
         *(
             f"bot PRs may only change the cost map files, not {path}"
             for path in changed_files
-            if path not in GUARDED_PATHS
+            if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
         ),
         *(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
         *(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),

ci_cd/generate_model_prices_schema.py

@@ -1,418 +0,0 @@
-from __future__ import annotations
-
-import json
-import sys
-from pathlib import Path
-from typing import Final, Optional
-
-import jsonschema
-
-REPO_ROOT = Path(__file__).parent.parent
-PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
-SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
-
-SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
-
-JsonSchema = dict
-
-NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
-NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
-BOOLEAN: JsonSchema = {"type": "boolean"}
-STRING: JsonSchema = {"type": "string"}
-TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"}
-WEEKDAY_PATTERN: Final = (
-    r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
-)
-
-EXTRA_BOOLEAN_KEYS = frozenset(
-    {
-        "gemini_native_audio",
-        "gemini_audio_only_live",
-        "uses_embed_content",
-        "use_openai_responses_path",
-        "bedrock_converse_supports_strict_tools",
-        "thinking_always_on",
-    }
-)
-
-HOURS_UTC: Final[JsonSchema] = {
-    "description": 'UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.',
-    "oneOf": [TIME_WINDOW, {"type": "array", "items": TIME_WINDOW, "minItems": 1}],
-}
-
-OFF_PEAK_WINDOW: Final[JsonSchema] = {
-    "type": "object",
-    "properties": {
-        "hours_utc": HOURS_UTC,
-        "weekdays": {
-            "type": "array",
-            "description": "ISO-8601 weekday numbers (1 = Monday .. 7 = Sunday) or English day names the window applies on.",
-            "items": {
-                "oneOf": [
-                    {"type": "integer", "minimum": 1, "maximum": 7},
-                    {"type": "string", "pattern": WEEKDAY_PATTERN},
-                ]
-            },
-            "minItems": 1,
-        },
-    },
-    "required": ["hours_utc"],
-    "additionalProperties": False,
-}
-
-OBJECT_KEYS: dict[str, JsonSchema] = {
-    "off_peak_pricing": {
-        "type": "object",
-        "description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.",
-        "properties": {
-            "hours_utc": HOURS_UTC,
-            "windows": {"type": "array", "items": OFF_PEAK_WINDOW, "minItems": 1},
-            "weekday_timezone": {
-                "type": "string",
-                "description": "IANA zone the weekdays of each window are read on; defaults to UTC.",
-            },
-            "input_cost_per_token": NONNEG_NUMBER,
-            "output_cost_per_token": NONNEG_NUMBER,
-            "output_cost_per_reasoning_token": NONNEG_NUMBER,
-            "cache_read_input_token_cost": NONNEG_NUMBER,
-            "cache_creation_input_token_cost": NONNEG_NUMBER,
-        },
-        "anyOf": [{"required": ["hours_utc"]}, {"required": ["windows"]}],
-        "additionalProperties": False,
-    },
-    "search_context_cost_per_query": {
-        "type": "object",
-        "description": "USD cost per web search query, keyed by search context size.",
-        "properties": {
-            "search_context_size_low": NONNEG_NUMBER,
-            "search_context_size_medium": NONNEG_NUMBER,
-            "search_context_size_high": NONNEG_NUMBER,
-        },
-        "additionalProperties": False,
-    },
-    "guardrail_cost_per_unit": {
-        "type": "object",
-        "description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",
-        "additionalProperties": NONNEG_NUMBER,
-    },
-    "metadata": {
-        "type": "object",
-        "description": "Free-form notes about the entry (e.g. pricing derivation).",
-    },
-    "provider_specific_entry": {
-        "type": "object",
-        "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
-    },
-}
-
-ARRAY_KEYS: dict[str, JsonSchema] = {
-    "supported_audio_formats": {
-        "type": "array",
-        "description": "Audio container formats the model can return.",
-        "items": {"type": "string", "enum": ["mp3", "wav"]},
-    },
-    "supported_endpoints": {
-        "type": "array",
-        "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
-        "items": STRING,
-    },
-    "supported_modalities": {
-        "type": "array",
-        "description": "Input modalities the model accepts.",
-        "items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
-    },
-    "supported_output_modalities": {
-        "type": "array",
-        "description": "Output modalities the model can produce.",
-        "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
-    },
-    "reasoning_effort_levels": {
-        "type": "array",
-        "description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
-        "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
-    },
-    "supported_regions": {
-        "type": "array",
-        "description": "Cloud regions the model is available in ('global' or region ids).",
-        "items": STRING,
-    },
-    "tiered_pricing": {
-        "type": "array",
-        "description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
-        "items": {
-            "type": "object",
-            "properties": {
-                "range": {
-                    "type": "array",
-                    "description": "[min, max] prompt-token span this tier applies to.",
-                    "items": NONNEG_NUMBER,
-                    "minItems": 2,
-                    "maxItems": 2,
-                },
-                "max_results_range": {
-                    "type": "array",
-                    "description": "[min, max] result-count span this tier applies to (search models).",
-                    "items": NONNEG_NUMBER,
-                    "minItems": 2,
-                    "maxItems": 2,
-                },
-                "input_cost_per_token": NONNEG_NUMBER,
-                "output_cost_per_token": NONNEG_NUMBER,
-                "output_cost_per_reasoning_token": NONNEG_NUMBER,
-                "cache_read_input_token_cost": NONNEG_NUMBER,
-                "cache_creation_input_token_cost": NONNEG_NUMBER,
-                "input_cost_per_query": NONNEG_NUMBER,
-            },
-            "additionalProperties": False,
-        },
-    },
-}
-
-INTEGER_KEYS: dict[str, JsonSchema] = {
-    "max_tokens": {
-        **NONNEG_INTEGER,
-        "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
-    },
-    "max_input_tokens": {
-        **NONNEG_INTEGER,
-        "description": "Maximum prompt/context tokens the model accepts.",
-    },
-    "max_output_tokens": {
-        **NONNEG_INTEGER,
-        "description": "Maximum tokens the model can generate in one response.",
-    },
-    "output_vector_size": {
-        **NONNEG_INTEGER,
-        "description": "Embedding dimension for embedding models.",
-    },
-    "prompt_cache_min_tokens": {
-        **NONNEG_INTEGER,
-        "description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
-    },
-    "token_count_framing": {
-        **NONNEG_INTEGER,
-        "description": "Tokens added once per request to a local count estimate, for the provider's chat framing.",
-    },
-    "tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
-    "rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
-}
-
-NUMBER_KEYS: dict[str, JsonSchema] = {
-    "regional_processing_uplift_multiplier_eu": {
-        "type": "number",
-        "minimum": 1,
-        "description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
-    },
-    "regional_processing_uplift_multiplier_us": {
-        "type": "number",
-        "minimum": 1,
-        "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
-    },
-    "regional_endpoint_uplift_multiplier": {
-        "type": "number",
-        "minimum": 1,
-        "description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).",
-    },
-}
-
-COST_DESCRIPTIONS: dict[str, str] = {
-    "input_cost_per_token": "USD per prompt token.",
-    "output_cost_per_token": "USD per generated token.",
-    "output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
-    "google_maps_grounding_cost_per_query": (
-        "USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
-    ),
-    "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
-    "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
-    "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
-    "output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
-}
-
-
-def cost_description(key: str) -> Optional[str]:
-    if key in COST_DESCRIPTIONS:
-        return COST_DESCRIPTIONS[key]
-    if key.endswith("_flex"):
-        return "Flex service-tier rate for the same-named base field."
-    if key.endswith("_priority"):
-        return "Priority service-tier rate for the same-named base field."
-    if "_above_" in key:
-        return "Rate applied once the prompt exceeds the token threshold in the field name."
-    return None
-
-
-def cost_schema(key: str) -> JsonSchema:
-    description = cost_description(key)
-    return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
-
-
-def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
-    return {
-        "litellm_provider": {
-            "type": "string",
-            "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
-        },
-        "mode": {
-            "type": "string",
-            "description": "Primary API surface / task type of the model.",
-            "enum": list(modes),
-        },
-        "source": {
-            "type": "string",
-            "description": "URL of the provider pricing/model page this entry was taken from.",
-        },
-        "deprecation_date": {
-            "type": "string",
-            "description": "Date the provider deprecates the model, YYYY-MM-DD.",
-            "format": "date",
-            "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$",
-        },
-        "web_search_billing_unit": {
-            "type": "string",
-            "description": "Whether web search is billed per query or per prompt.",
-            "enum": ["per_query", "per_prompt"],
-        },
-        "bedrock_output_config_effort_ceiling": {
-            "type": "string",
-            "description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
-            "enum": ["low", "medium", "high", "max", "xhigh"],
-        },
-        "default_reasoning_effort": {
-            "type": "string",
-            "description": (
-                "Reasoning effort the provider applies when the request omits reasoning_effort. "
-                "Gates whether a non-default temperature or the top_p/logprobs sampling params are "
-                "accepted, which hold only when the effort resolves to 'none'."
-            ),
-            "enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
-        },
-        "comment": STRING,
-        "token_count_fallback_model": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Model that counts tokens for this one when the provider cannot count it.",
-        },
-        "audio_transcription_config": STRING,
-        "vertex_ai_audio_api": {
-            "type": "string",
-            "enum": ["lyria_predict", "lyria_interactions"],
-        },
-    }
-
-
-def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
-    curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
-    if key in curated:
-        return curated[key]
-    if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
-        return BOOLEAN
-    if "cost" in key:
-        return cost_schema(key)
-    return None
-
-
-def build_schema(prices: dict) -> JsonSchema:
-    entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
-    all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
-    modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
-    unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
-    if unclassified:
-        raise SystemExit(
-            f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
-            f"Add them to the key tables in {Path(__file__).name} and rerun it."
-        )
-    entry_properties = {key: classify(key, modes) for key in all_keys}
-    return {
-        "$schema": "https://json-schema.org/draft/2020-12/schema",
-        "title": "LiteLLM model_prices_and_context_window.json",
-        "description": (
-            "Schema for LiteLLM's model price and context window registry "
-            "(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
-            "Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
-            "optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
-            "All costs are USD per unit. New optional fields are added regularly, so consumers should "
-            "ignore unknown fields rather than reject them."
-        ),
-        "type": "object",
-        "properties": {
-            "sample_spec": {
-                "type": "object",
-                "description": (
-                    "Documentation placeholder illustrating the entry shape; not a real model and not "
-                    "schema-conformant (several values are prose)."
-                ),
-            },
-            "fallback_generalizations": {
-                "type": "object",
-                "description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
-                "properties": {
-                    "rules": {
-                        "type": "array",
-                        "items": {
-                            "type": "object",
-                            "properties": {
-                                "name": STRING,
-                                "pattern": STRING,
-                                "description": STRING,
-                            },
-                            "required": ["name", "pattern"],
-                            "additionalProperties": True,
-                        },
-                    }
-                },
-                "additionalProperties": False,
-            },
-        },
-        "additionalProperties": {"$ref": "#/$defs/modelEntry"},
-        "$defs": {
-            "modelEntry": {
-                "type": "object",
-                "description": (
-                    "Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
-                    "are optional; boolean capability flags are simply omitted when unknown or false."
-                ),
-                "required": ["litellm_provider"],
-                "properties": entry_properties,
-                "additionalProperties": True,
-            }
-        },
-    }
-
-
-def render(schema: JsonSchema) -> str:
-    return json.dumps(schema, indent=2) + "\n"
-
-
-def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
-    validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
-    return tuple(
-        f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
-        for error in validator.iter_errors(prices)
-    )
-
-
-def main() -> int:
-    check = "--check" in sys.argv[1:]
-    prices = json.loads(PRICES_PATH.read_text())
-    rendered = render(build_schema(prices))
-    errors = validation_errors(prices, json.loads(rendered))
-    if errors:
-        print(f"{PRICES_PATH.name} does not validate against the generated schema:")
-        print("\n".join(errors[:20]))
-        return 1
-    if not check:
-        SCHEMA_PATH.write_text(rendered)
-        print(f"wrote {SCHEMA_PATH}")
-        return 0
-    if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
-        print(
-            f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
-            f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
-        )
-        return 1
-    print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

db_scripts/strip_vector_store_internal_metadata.py

@@ -0,0 +1,201 @@
+import argparse
+import os
+import sys
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from itertools import accumulate, repeat, takewhile
+from typing import Final, Literal, TextIO
+
+import httpx
+from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
+
+from litellm.vector_stores.utils import is_litellm_written_metadata_key
+
+DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
+LIST_PAGE_SIZE: Final = "100"
+
+
+class VectorStore(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    id: str
+    metadata: dict[str, JsonValue] | None = None
+
+
+class VectorStorePage(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    data: tuple[VectorStore, ...]
+    has_more: bool = False
+
+
+class OpenAIErrorBody(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    message: str = ""
+
+
+class OpenAIErrorResponse(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    error: OpenAIErrorBody
+
+
+@dataclass(frozen=True, slots=True)
+class Options:
+    api_key: str
+    api_base: str
+    vector_store_ids: tuple[str, ...]
+    extra_keys: frozenset[str]
+    apply: bool
+
+
+@dataclass(frozen=True, slots=True)
+class StoreResult:
+    store_id: str
+    matched_keys: tuple[str, ...]
+    status: Literal["dry_run", "updated", "failed"]
+    error: str = ""
+
+
+def is_proxy_written_key(key: str, extra_keys: frozenset[str]) -> bool:
+    return is_litellm_written_metadata_key(key) or key in extra_keys
+
+
+def matching_keys(store: VectorStore, extra_keys: frozenset[str]) -> tuple[str, ...]:
+    return tuple(sorted(key for key in (store.metadata or {}) if is_proxy_written_key(key, extra_keys)))
+
+
+def fetch_page(client: httpx.Client, after: str | None) -> VectorStorePage:
+    params: Final = {"limit": LIST_PAGE_SIZE} if after is None else {"limit": LIST_PAGE_SIZE, "after": after}
+    return VectorStorePage.model_validate_json(client.get("/vector_stores", params=params).raise_for_status().content)
+
+
+def fetch_next_page(client: httpx.Client, page: VectorStorePage | None) -> VectorStorePage | None:
+    if page is None or not page.has_more or not page.data:
+        return None
+    return fetch_page(client, after=page.data[-1].id)
+
+
+def list_all_stores(client: httpx.Client) -> Iterator[VectorStore]:
+    pages: Final = accumulate(
+        repeat(None), lambda page, _: fetch_next_page(client, page), initial=fetch_page(client, after=None)
+    )
+    for page in takewhile(lambda page: page is not None, pages):
+        yield from page.data
+
+
+def retrieve_stores(client: httpx.Client, store_ids: tuple[str, ...]) -> Iterator[VectorStore]:
+    for store_id in store_ids:
+        yield VectorStore.model_validate_json(client.get(f"/vector_stores/{store_id}").raise_for_status().content)
+
+
+def describe_http_error(response: httpx.Response) -> str:
+    try:
+        message: Final = OpenAIErrorResponse.model_validate_json(response.content).error.message
+    except ValidationError:
+        return f"HTTP {response.status_code}"
+    return f"HTTP {response.status_code}: {message}"
+
+
+def delete_keys(client: httpx.Client, store_id: str, keys: tuple[str, ...]) -> StoreResult:
+    try:
+        response: Final = client.post(f"/vector_stores/{store_id}", json={"metadata": dict.fromkeys(keys)})
+    except httpx.TransportError as error:
+        return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=type(error).__name__)
+    if response.is_success:
+        return StoreResult(store_id=store_id, matched_keys=keys, status="updated")
+    return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=describe_http_error(response))
+
+
+def clean_store(client: httpx.Client, store: VectorStore, options: Options) -> StoreResult | None:
+    keys: Final = matching_keys(store, options.extra_keys)
+    if not keys:
+        return None
+    if not options.apply:
+        return StoreResult(store_id=store.id, matched_keys=keys, status="dry_run")
+    return delete_keys(client, store.id, keys)
+
+
+def format_result(result: StoreResult) -> str:
+    key_list: Final = ", ".join(result.matched_keys)
+    count: Final = len(result.matched_keys)
+    match result.status:
+        case "dry_run":
+            return f"{result.store_id}: would delete {count} keys: {key_list}"
+        case "updated":
+            return f"{result.store_id}: deleted {count} keys: {key_list}"
+        case "failed":
+            return f"{result.store_id}: failed to delete {count} keys ({result.error}): {key_list}"
+
+
+def clean_and_report(
+    client: httpx.Client, stores: Iterator[VectorStore], options: Options, out: TextIO
+) -> Iterator[StoreResult | None]:
+    for store in stores:
+        result: Final = clean_store(client, store, options)
+        if result is not None:
+            print(format_result(result), file=out)
+        yield result
+
+
+def run(options: Options, client: httpx.Client, out: TextIO) -> int:
+    stores: Final = (
+        retrieve_stores(client, options.vector_store_ids) if options.vector_store_ids else list_all_stores(client)
+    )
+    scanned: Final = tuple(clean_and_report(client, stores, options, out))
+    results: Final = tuple(result for result in scanned if result is not None)
+    updated: Final = sum(1 for result in results if result.status == "updated")
+    failed: Final = sum(1 for result in results if result.status == "failed")
+    mode: Final = "applied" if options.apply else "dry run, pass --apply to delete"
+    print(
+        f"scanned={len(scanned)} matched={len(results)} updated={updated} failed={failed} ({mode})",
+        file=out,
+    )
+    return 1 if failed else 0
+
+
+def parse_options(argv: Sequence[str] | None) -> Options:
+    parser: Final = argparse.ArgumentParser(
+        description="Remove LiteLLM proxy-written keys from OpenAI vector store metadata. Dry run by default."
+    )
+    parser.add_argument("--api-key", default=os.environ.get("OPENAI_API_KEY"), help="defaults to $OPENAI_API_KEY")
+    parser.add_argument("--api-base", default=DEFAULT_API_BASE)
+    parser.add_argument("--vector-store-id", action="append", default=[], help="repeatable, defaults to every store")
+    parser.add_argument("--extra-key", action="append", default=[], help="repeatable, another key name to delete")
+    parser.add_argument("--apply", action="store_true", help="delete the matching keys instead of listing them")
+    args: Final = parser.parse_args(argv)
+    if not args.api_key:
+        parser.error("--api-key or OPENAI_API_KEY is required")
+    return Options(
+        api_key=args.api_key,
+        api_base=args.api_base.rstrip("/"),
+        vector_store_ids=tuple(args.vector_store_id),
+        extra_keys=frozenset(args.extra_key),
+        apply=args.apply,
+    )
+
+
+def main(argv: Sequence[str] | None = None, transport: httpx.BaseTransport | None = None) -> int:
+    options: Final = parse_options(argv)
+    with httpx.Client(
+        base_url=options.api_base,
+        headers={"Authorization": f"Bearer {options.api_key}"},
+        timeout=60.0,
+        transport=transport,
+    ) as client:
+        try:
+            return run(options, client, sys.stdout)
+        except httpx.HTTPStatusError as error:
+            sys.stderr.write(f"could not read vector stores: {describe_http_error(error.response)}\n")
+            return 2
+        except httpx.TransportError as error:
+            sys.stderr.write(f"could not reach {options.api_base}: {type(error).__name__}\n")
+            return 2
+        except ValidationError:
+            sys.stderr.write(f"unexpected vector store response shape from {options.api_base}\n")
+            return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -3,6 +3,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
 """
 
 import asyncio
+import json
 from collections.abc import Mapping, Sequence
 from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
@@ -227,7 +228,7 @@ class CheckBatchCost:
             "so it will no longer be polled"
         )
 
-    async def _claim_job_for_costing(self, job: "ManagedObjectRow") -> bool:
+    async def _claim_job_for_costing(self, job: "ManagedObjectRow", response: "LiteLLMBatch") -> bool:
         """
         Atomically flip batch_processed from false to true, returning whether this pod won
         the row. Every pod and uvicorn worker schedules its own poller against the shared
@@ -241,10 +242,16 @@ class CheckBatchCost:
         """
         if not self._has_batch_processed_column:
             return True
+        from litellm_enterprise.proxy.hooks.managed_files import postgres_storable_json
+
         try:
             claimed: Final = await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": False},
-                data={"batch_processed": True},
+                data={
+                    "batch_processed": True,
+                    "status": response.status if response.status != "completed" else "complete",
+                    "file_object": postgres_storable_json(response),
+                },
             )
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -253,7 +260,9 @@ class CheckBatchCost:
             return False
         return claimed > 0
 
-    async def _release_job_claim(self, job: "ManagedObjectRow") -> None:
+    async def _release_job_claim(
+        self, job: "ManagedObjectRow", original_status: str | None, original_file_object: object
+    ) -> None:
         """Give a claimed row back once billing it failed, so a later poll cycle retries it.
 
         Safe to match on batch_processed=True: while this poller is active the retrieve
@@ -265,7 +274,11 @@ class CheckBatchCost:
         try:
             await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": True},
-                data={"batch_processed": False},
+                data={
+                    "batch_processed": False,
+                    "status": original_status,
+                    "file_object": json.dumps(original_file_object),
+                },
             )
         except Exception as db_err:
             verbose_proxy_logger.error(
@@ -564,8 +577,6 @@ class CheckBatchCost:
 
     @staticmethod
     def _get_input_file_id(job: "ManagedObjectRow") -> Optional[str]:
-        import json
-
         from litellm.types.utils import LiteLLMBatch
 
         file_object: Final = job.file_object
@@ -608,7 +619,11 @@ class CheckBatchCost:
         from litellm.files.types import FileContentCallOptions
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
         from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
-        from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info, mask_api_base_credentials
+        from litellm.litellm_core_utils.litellm_logging import (
+            deployment_pricing_model_info,
+            mask_api_base_credentials,
+            resolve_vertex_location_for_cost,
+        )
         from litellm.proxy.openai_files_endpoints.common_utils import (
             _is_base64_encoded_unified_file_id,
         )
@@ -718,8 +733,7 @@ class CheckBatchCost:
                         )
                     except Exception as _e:
                         verbose_proxy_logger.warning(
-                            f"CheckBatchCost: failed to create managed file ID for "
+                            f"CheckBatchCost: failed to create managed file ID for {_file_attr}={_raw_file_id!r}: {_e}"
-                            f"{_file_attr}={_raw_file_id!r}: {_e}"
                         )
 
         # Pass the deployment's router-registered pricing (litellm_params custom
@@ -738,6 +752,12 @@ class CheckBatchCost:
             custom_llm_provider=batch_file_provider,
             model_name=model_name,
             model_info=deployment_model_info,
+            vertex_location=resolve_vertex_location_for_cost(
+                custom_llm_provider=llm_provider,
+                litellm_params=deployment_info.litellm_params.model_dump(exclude_none=True),
+                optional_params=None,
+                model=model_name,
+            ),
         )
         error_file_failed_requests: Final = await count_error_file_failed_requests(
             response,
@@ -755,8 +775,9 @@ class CheckBatchCost:
                 failed_requests=output_file_result.failed_requests + error_file_failed_requests,
             )
         )
+        logging_model: Final = batch_result.models[0] if batch_result.models else litellm_model_name or model_id
         logging_obj = LiteLLMLogging(
-            model=batch_result.models[0],
+            model=logging_model,
             messages=[{"role": "user", "content": "<retrieve_batch>"}],
             stream=False,
             call_type="aretrieve_batch",
@@ -787,7 +808,9 @@ class CheckBatchCost:
             custom_llm_provider=str(llm_provider) if llm_provider else None,
         )
 
-        if not await self._claim_job_for_costing(job):
+        original_status: Final = job.status
+        original_file_object: Final = job.file_object
+        if not await self._claim_job_for_costing(job, response.model_copy(update={"usage": batch_result.usage})):
             verbose_proxy_logger.info(
                 f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
                 "in this window, so its cost is already being tracked there"
@@ -806,7 +829,7 @@ class CheckBatchCost:
                 batch_completion_cost=batch_result.completion_cost,
             )
         except Exception:
-            await self._release_job_claim(job)
+            await self._release_job_claim(job, original_status, original_file_object)
             raise
 
         # Record batch duration (completed_at - created_at)
@@ -876,7 +899,10 @@ class CheckBatchCost:
         )
         from litellm.batches.batch_utils import fetch_batch_managed_file_content
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
-        from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info
+        from litellm.litellm_core_utils.litellm_logging import (
+            deployment_pricing_model_info,
+            resolve_vertex_location_for_cost,
+        )
 
         try:
             deployment_info: Final = self.llm_router.get_deployment(model_id=model_id)
@@ -920,6 +946,12 @@ class CheckBatchCost:
                 cost_provider=cost_provider,
                 model_info=deployment_pricing_model_info(model_id=model_id, deployment_model=litellm_model_name),
                 metadata=line_item_metadata(await self._creator_attribution.batch_metadata(job, batch_id)),
+                vertex_location=resolve_vertex_location_for_cost(
+                    custom_llm_provider=llm_provider,
+                    litellm_params=deployment_info.litellm_params.model_dump(exclude_none=True),
+                    optional_params=None,
+                    model=model_name,
+                ),
             )
             input_content: Final = (
                 b""
@@ -1099,10 +1131,7 @@ class CheckBatchCost:
                 continue
 
             ## RETRIEVE THE BATCH JOB OUTPUT FILE
-            if (
+            if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None:
-                response.status in PROVIDER_TERMINAL_BATCH_STATUSES
-                and response.output_file_id is not None
-            ):
                 try:
                     tracked = await self._track_completed_batch_cost(
                         job=job,
@@ -1133,24 +1162,19 @@ class CheckBatchCost:
                 # Track this job for the final metrics summary
                 processed_models.append(tracked)
 
-                # mark the job as complete
+                if not self._has_batch_processed_column:
-                try:
+                    try:
-                    update_data: dict = {
+                        await _managed_object_table(self.prisma_client).update(
-                        "status": response.status if response.status != "completed" else "complete",
+                            where={"id": job.id},
-                        "file_object": response.model_dump_json(),
+                            data={
-                    }
+                                "status": response.status if response.status != "completed" else "complete",
-                    if self._has_batch_processed_column:
+                                "file_object": response.model_dump_json(),
-                        update_data["batch_processed"] = True
+                            },
-                    await _managed_object_table(self.prisma_client).update(
+                        )
-                        where={"id": job.id},
+                    except Exception as db_err:
-                        data=update_data,
+                        verbose_proxy_logger.error(
-                    )
+                            f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
-                except Exception as db_err:
+                        )
-                    verbose_proxy_logger.error(
-                        f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
-                    )
-                    if not self._has_batch_processed_column:
-                        # nothing claimed the row, so it stays in the poll page and its lines wait for that retry
                         continue
                 self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)
 

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -6,6 +6,7 @@ import base64
 import json
 import time
 from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from itertools import chain
 from types import MappingProxyType
 from typing import (
     TYPE_CHECKING,
@@ -416,7 +417,7 @@ def _storable_json_value(value: JsonValue) -> JsonValue:
     return value
 
 
-def _postgres_storable_json(file_object: BaseModel) -> str:
+def postgres_storable_json(file_object: BaseModel) -> str:
     dumped: Final = file_object.model_dump_json()
     if "\\u0000" not in dumped:
         return dumped
@@ -479,6 +480,13 @@ def _with_managed_background_marker(request: Mapping[str, object]) -> dict[str,
 
 
 _MANAGED_FILES_TARGET: Final = "managed_files"
+_PROVIDER_FILE_ID_KWARGS: Final = MappingProxyType(
+    {
+        CallTypes.acreate_batch: ("input_file_id",),
+        CallTypes.acreate_fine_tuning_job: ("training_file", "validation_file"),
+    }
+)
+_PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDER_FILE_ID_KWARGS.values())))
 _PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
 
 
@@ -650,7 +658,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         )
         # FIX: Update status and file_object on every operation to keep state in sync
         update_columns: Final = {
-            "file_object": _postgres_storable_json(stored_object),
+            "file_object": postgres_storable_json(stored_object),
             "status": file_object.status,
             "updated_by": user_api_key_dict.user_id,
         }
@@ -666,7 +674,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             data={
                 "create": {
                     "unified_object_id": unified_object_id,
-                    "file_object": _postgres_storable_json(stored_object),
+                    "file_object": postgres_storable_json(stored_object),
                     "model_object_id": model_object_id,
                     "file_purpose": file_purpose,
                     "created_by": resolve_resource_owner_id(user_api_key_dict),
@@ -702,7 +710,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "unified_object_id": response_id,
                     "model_object_id": decrypted_response_id(response_id),
                     "file_purpose": "response",
-                    "file_object": _postgres_storable_json(_without_model_output(response)),
+                    "file_object": postgres_storable_json(_without_model_output(response)),
                     "status": PENDING_RESPONSE_STATUSES[0],
                     "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
@@ -1562,6 +1570,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 if isinstance(file_id, str) and file_id
             )
             await self.check_file_ids_access(fine_tuning_file_ids, user_api_key_dict)
+            if fine_tuning_file_ids:
+                data["model_file_id_mapping"] = await self.get_model_file_id_mapping(
+                    fine_tuning_file_ids, user_api_key_dict.parent_otel_span
+                )
 
         return data
 
@@ -1576,24 +1588,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         if request_kwargs is None:
             return healthy_deployments
 
-        input_file_id = cast(Optional[str], request_kwargs.get("input_file_id"))
+        request: Final = cast(Mapping[str, object], request_kwargs)
-        model_file_id_mapping = cast(
+        model_file_id_mapping: Final = cast(Mapping[str, Mapping[str, str]], request.get("model_file_id_mapping") or {})
-            Optional[Dict[str, Dict[str, str]]],
+        requested_file_ids: Final = (request.get(key) for key in _PROVIDER_FILE_ID_KEYS)
-            request_kwargs.get("model_file_id_mapping"),
+        model_ids_per_file: Final = tuple(
+            frozenset(model_file_id_mapping[file_id])
+            for file_id in requested_file_ids
+            if isinstance(file_id, str) and file_id in model_file_id_mapping
+        )
+        allowed_model_ids: Final[frozenset[str]] = (
+            model_ids_per_file[0].intersection(*model_ids_per_file[1:]) if model_ids_per_file else frozenset()
         )
-        allowed_model_ids = []
-        if input_file_id and model_file_id_mapping:
-            model_id_dict = model_file_id_mapping.get(input_file_id, {})
-            allowed_model_ids = list(model_id_dict.keys())
 
         if len(allowed_model_ids) == 0:
             return healthy_deployments
 
-        return [
+        holders: Final[list[dict[str, object]]] = [
             deployment
             for deployment in healthy_deployments
             if deployment.get("model_info", {}).get("id") in allowed_model_ids
         ]
+        return holders if holders or "input_file_id" in request else healthy_deployments
 
     async def async_pre_call_deployment_hook(
         self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
@@ -1601,25 +1616,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         """
         Allow modifying the request just before it's sent to the deployment.
         """
-        accessor_key: Optional[str] = None
+        accessor_keys: Final = _PROVIDER_FILE_ID_KWARGS.get(call_type) if call_type else None
-        if call_type and call_type == CallTypes.acreate_batch:
+        if accessor_keys is None:
-            accessor_key = "input_file_id"
-        elif call_type and call_type == CallTypes.acreate_fine_tuning_job:
-            accessor_key = "training_file"
-        else:
             return kwargs
 
-        if accessor_key:
+        model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
+        # model_info may be at top-level or nested under litellm_metadata
+        # (batch/file operations use litellm_metadata)
+        model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
+        if model_id is None:
+            model_id = cast(
+                Optional[str],
+                kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
+            )
+        for accessor_key in accessor_keys:
             input_file_id = cast(Optional[str], kwargs.get(accessor_key))
-            model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
-            # model_info may be at top-level or nested under litellm_metadata
-            # (batch/file operations use litellm_metadata)
-            model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
-            if model_id is None:
-                model_id = cast(
-                    Optional[str],
-                    kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
-                )
             mapped_file_id: Optional[str] = None
             if input_file_id and model_file_id_mapping and model_id:
                 mapped_file_id = model_file_id_mapping.get(input_file_id, {}).get(model_id, None)
@@ -1813,7 +1824,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     ),
                 )
 
-    async def get_model_file_id_mapping(self, file_ids: List[str], litellm_parent_otel_span: Span) -> dict:
+    async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict:
         """
         Get model-specific file IDs for a list of proxy file IDs.
         Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.77"
+version = "0.1.78"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.77"
+version = "0.1.78"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -20,6 +20,33 @@ RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${
     make -j"$(nproc)" pgbouncer && \
     install -m 0755 pgbouncer /usr/local/bin/pgbouncer
 
+# ---------- Rust extension ----------
+# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
+FROM $LITELLM_BUILD_IMAGE AS rust-native
+
+WORKDIR /app
+USER root
+
+COPY --from=uvbin /uv /uvx /usr/local/bin/
+
+RUN for i in 1 2 3; do \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
+      [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
+      sleep 5; \
+    done
+
+ENV UV_PYTHON_DOWNLOADS=0
+
+COPY pyproject.toml README.md rust-toolchain.toml ./
+COPY .cargo ./.cargo
+COPY litellm-rust ./litellm-rust
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    mkdir -p litellm/rust_bridge /wheels && \
+    touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
+    uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
+        python -c "import maturin; maturin.build_wheel('/wheels')"
+
 # ---------- Builder ----------
 FROM $LITELLM_BUILD_IMAGE AS builder
 
@@ -66,11 +93,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra bedrock-realtime \
         --python python3.13
 
-# Stage 2 — copy source and install the project + workspace members.
+# Stage 2 — copy source and install the workspace members.
 COPY . .
 
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv sync --frozen --no-default-groups --no-editable \
+    uv sync --frozen --no-install-project --no-default-groups --no-editable \
         --extra proxy \
         --extra proxy-runtime \
         --group admin-mcp \
@@ -79,6 +106,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra bedrock-realtime \
         --python python3.13
 
+RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
+    uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
+
 # PYTHONPATH=/app makes the source tree shadow the installed package, so the
 # compiled Rust extension must live next to the source or it is never imported.
 RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/

helm/litellm-helm/README.md

@@ -70,11 +70,18 @@ proxyConfigMap:
 proxy_config:
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
+  model_list: []
+```
+
+The chart ships no models, so add them under `proxy_config.model_list` or from the Admin UI with `store_model_in_db: true`
+
+```
+proxy_config:
   model_list:
-    - model_name: gpt-3.5-turbo
+    - model_name: gpt-5.6
       litellm_params:
-        model: gpt-3.5-turbo
+        model: openai/gpt-5.6
-        api_key: eXaMpLeOnLy
+        api_key: os.environ/OPENAI_API_KEY
 ```
 
 #### Example using existing `proxyConfigMap` instead of creating it:

helm/litellm-helm/tests/deployment_tests.yaml

@@ -270,6 +270,32 @@ tests:
       - matchRegex:
           path: data["config.yaml"]
           pattern: '(?m)^\s*health_check_details:\s*false$'
+  - it: should not render any placeholder models when the user sets proxy_config without model_list
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.general_settings.store_model_in_db: true
+    asserts:
+      - notMatchRegex:
+          path: data["config.yaml"]
+          pattern: 'model_name:'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'store_model_in_db: true'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'master_key: os.environ/PROXY_MASTER_KEY'
+  - it: should render a user-supplied model_list as the only models
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.model_list:
+        - model_name: my-model
+          litellm_params:
+            model: openai/my-model
+            api_key: os.environ/OPENAI_API_KEY
+    asserts:
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'model_list:\n- litellm_params:\n    api_key: os.environ/OPENAI_API_KEY\n    model: openai/my-model\n  model_name: my-model\n$'
   - it: should allow overriding liveness, readiness, and startup probes
     template: deployment.yaml
     set:

helm/litellm-helm/values.yaml

@@ -180,17 +180,7 @@ proxyConfigMap:
 #  Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
 #  Reference: https://docs.litellm.ai/docs/proxy/configs
 proxy_config:
-  model_list:
+  model_list: []
-    # At least one model must exist for the proxy to start.
-    - model_name: gpt-3.5-turbo
-      litellm_params:
-        model: gpt-3.5-turbo
-        api_key: eXaMpLeOnLy
-    - model_name: fake-openai-endpoint
-      litellm_params:
-        model: openai/fake
-        api_key: fake-key
-        api_base: https://exampleopenaiendpoint-production.up.railway.app/
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
 

litellm-proxy-extras/litellm_proxy_extras/migrations/20261015120000_add_environment_variables_revision_trigger/migration.sql

@@ -0,0 +1,23 @@
+SET LOCAL lock_timeout = '10s';
+
+CREATE SEQUENCE IF NOT EXISTS "litellm_environment_variables_revision_seq";
+GRANT USAGE ON SEQUENCE "litellm_environment_variables_revision_seq" TO PUBLIC;
+
+CREATE OR REPLACE FUNCTION litellm_bump_environment_variables_revision() RETURNS TRIGGER AS $$
+DECLARE
+  next_revision BIGINT := nextval(format('%I.litellm_environment_variables_revision_seq', TG_TABLE_SCHEMA)::regclass);
+BEGIN
+  IF TG_OP = 'UPDATE' THEN
+    NEW."reload_revision" := GREATEST(next_revision, OLD."reload_revision" + 1);
+  ELSE
+    NEW."reload_revision" := next_revision;
+  END IF;
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS "litellm_environment_variables_revision" ON "LiteLLM_Config";
+CREATE TRIGGER "litellm_environment_variables_revision"
+BEFORE INSERT OR UPDATE ON "LiteLLM_Config"
+FOR EACH ROW WHEN (NEW."param_name" = 'environment_variables')
+EXECUTE PROCEDURE litellm_bump_environment_variables_revision();

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.109"
+version = "0.4.110"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.109"
+version = "0.4.110"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/Cargo.lock

@@ -4083,6 +4083,7 @@ dependencies = [
 name = "litellm-inference-responses"
 version = "0.1.0"
 dependencies = [
+ "base64 0.22.1",
  "bytes",
  "futures-util",
  "litellm-auth",

litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs

@@ -94,6 +94,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
     "docker_model_runner",
     "ragflow",
     "pinstripes",
+    "openinfer",
     "darkbloom",
     "meta",
     "cognition",

litellm-rust/crates/inference-chat/src/common_utils.rs

@@ -6,9 +6,11 @@ use litellm_llms::{
     bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
     openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG,
 };
+use litellm_llms_types::formats::chat_completions::ChatMessage;
 use serde_json::{Map, Value};
 
 use super::Error;
+use crate::types::{ChatCompletionsCall, ResolvedChatCompletionsRequest};
 
 const HEADER_CONTEXT: &str = "chat completions";
 
@@ -42,3 +44,58 @@ pub(super) fn string_headers(
 ) -> Result<Vec<(String, String)>, Error> {
     shared_string_headers(HEADER_CONTEXT, extra_headers).map_err(Error::from)
 }
+
+pub(super) fn resolve_request(
+    call: ChatCompletionsCall,
+    config: &'static dyn BaseConfig,
+) -> Result<ResolvedChatCompletionsRequest, Error> {
+    let messages = serde_json::from_value::<Vec<ChatMessage>>(call.messages).map_err(|err| {
+        Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
+            "chat completions messages",
+            err,
+        ))
+    })?;
+    if messages.is_empty() {
+        return Err(Error::InvalidRequest(
+            "chat completions requires at least one message".into(),
+        ));
+    }
+    if let Some(reason) = config.unsupported_reason(&messages, &call.optional_params) {
+        return Err(Error::Unsupported(reason.0));
+    }
+    Ok(ResolvedChatCompletionsRequest {
+        messages,
+        optional_params: call.optional_params,
+        api_key: call.api_key,
+        api_base: call.api_base,
+        extra_headers: call.extra_headers,
+        timeout: call.timeout,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::chat_completions_provider;
+    use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+
+    #[rstest]
+    #[case::anthropic(LlmProviders::Anthropic, ("anthropic-version", "2023-06-01"))]
+    #[case::bedrock(LlmProviders::Bedrock, ("Content-Type", "application/json"))]
+    fn resolves_supported_provider_configs(
+        #[case] provider: LlmProviders,
+        #[case] expected_header: (&str, &str),
+    ) {
+        let config = chat_completions_provider(provider)
+            .expect("provider has a chat completions config")
+            .config();
+        assert!(config.default_headers().contains(&expected_header));
+    }
+
+    #[rstest]
+    #[case::openai(LlmProviders::Openai)]
+    fn rejects_providers_without_a_chat_config(#[case] provider: LlmProviders) {
+        assert!(chat_completions_provider(provider).is_none());
+    }
+}

litellm-rust/crates/inference-chat/src/handler.rs

@@ -3,44 +3,66 @@ use std::time::Duration;
 
 use litellm_auth::AuthServices;
 use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
-use litellm_http::{Client, outbound::OutboundRequest, request::truncate_error_body};
+use litellm_http::{
+    Client,
+    outbound::OutboundRequest,
+    request::{truncate_error_body, with_default_headers},
+};
 use litellm_llms::base_llm::{
-    auth::{Authenticated, resolve_auth},
+    auth::{Authenticated, ValidatedEnvironment, resolve_auth},
+    chat::transformation::BaseConfig,
     chat::transformation::ProviderChatResponseData,
 };
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
 use super::Error;
-use crate::{constants::CHAT_COMPLETIONS_TIMEOUT_SECS, types::ProviderChatCompletionsRequest};
+use crate::{
+    common_utils::string_headers, constants::CHAT_COMPLETIONS_TIMEOUT_SECS,
+    types::ResolvedChatCompletionsRequest,
+};
 
+#[expect(clippy::too_many_arguments)] // mirrors BaseLLMHTTPHandler.completion, which takes each stage input separately
 pub(super) async fn execute(
     http: &Client,
     auth: &AuthServices,
-    request: ProviderChatCompletionsRequest,
+    config: &'static dyn BaseConfig,
+    provider: litellm_inference::provider::ResolvedProvider<'_>,
+    call: ResolvedChatCompletionsRequest,
+    secrets: Secrets,
     cache: Option<litellm_cache_response::ScopedCache>,
     cache_options: Option<litellm_cache_response::CachePolicy>,
     interceptors: &impl Interceptors<Error>,
     observers: Option<&ObservationSender>,
 ) -> Result<ChatCompletionsResponse, Error> {
-    let ProviderChatCompletionsRequest {
+    let validated = config.validate_environment(
-        model,
+        string_headers(call.extra_headers)?,
-        custom_llm_provider,
+        call.api_key.as_deref(),
-        config,
+        provider.model,
-        url,
+        &call.optional_params,
-        body,
+        &|key| secrets.get(key),
-        optional_params,
+    )?;
-        environment,
+    let url = config.get_complete_url(
-        secrets,
+        call.api_base.as_deref(),
-        timeout,
+        provider.model,
-        api_key,
+        &call.optional_params,
-    } = request;
+        &|key| secrets.get(key),
+    )?;
+    let body = config
+        .transform_request(provider.model, call.messages, call.optional_params.clone())?
+        .body;
+    let environment = ValidatedEnvironment {
+        headers: with_default_headers(validated.headers, config.default_headers()),
+        auth: validated.auth,
+    };
+    let provider_name = <&'static str>::from(provider.provider);
     let context = RequestContext {
-        model: model.clone(),
+        model: provider.model.to_owned(),
-        custom_llm_provider,
+        custom_llm_provider: provider_name.to_owned(),
-        optional_params: Value::Object(optional_params),
+        optional_params: Value::Object(call.optional_params),
         secret_fields: Vec::new(),
-        api_key,
+        api_key: call.api_key.map(litellm_auth::SecretValue::new),
     };
     let authenticated = resolve_auth(auth, environment, &|key| secrets.get(key)).await?;
     let identity = litellm_host::interceptors::ProviderIdentity {
@@ -76,7 +98,7 @@ pub(super) async fn execute(
                 },
                 wire.url,
                 &wire.body,
-                timeout,
+                call.timeout,
             )?;
 
             let response = litellm_inference::outbound::send(outbound, http)
@@ -121,7 +143,7 @@ pub(super) async fn execute(
                 ))
             })?;
             config
-                .transform_response(&model, ProviderChatResponseData { body })
+                .transform_response(provider.model, ProviderChatResponseData { body })
                 .map_err(Error::from)
                 .map_err(as_response_error)
         },
@@ -134,7 +156,7 @@ pub(super) async fn execute(
 ///
 /// A config reports the same variants on either side of the call: a missing
 /// field or an unsupported block can mean "this request cannot be translated"
-/// during prepare and "this response cannot be normalized" here. Only the
+/// before the request is sent and "this response cannot be normalized" here. Only the
 /// second kind has already been billed, and a host that keeps a reference
 /// implementation must not retry those, so collapse them to one variant that
 /// can only mean the provider was already called.
@@ -178,9 +200,10 @@ mod tests {
 
     use super::*;
     use crate::{
-        prepare::{prepare_provider_request, resolve_request},
+        common_utils::{chat_completions_provider, resolve_request},
-        types::ChatCompletionsRequest,
+        types::{ChatCompletionsCall, ResolvedChatCompletionsRequest},
     };
+    use litellm_inference::provider::resolve_llm_provider;
 
     const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
 
@@ -217,22 +240,33 @@ mod tests {
         }
     }
 
-    fn prepared(api_base: &str) -> ProviderChatCompletionsRequest {
+    fn prepared(
-        prepare_provider_request(
+        api_base: &str,
-            resolve_request(ChatCompletionsRequest {
+    ) -> (
-                model: "anthropic/claude-sonnet-4-5",
+        litellm_inference::provider::ResolvedProvider<'static>,
+        &'static dyn BaseConfig,
+        ResolvedChatCompletionsRequest,
+    ) {
+        let provider =
+            resolve_llm_provider("anthropic/claude-sonnet-4-5", None, "chat completions").unwrap();
+        let config = chat_completions_provider(provider.provider)
+            .expect("provider has a chat completions config")
+            .config();
+        let request = resolve_request(
+            ChatCompletionsCall {
+                model: "anthropic/claude-sonnet-4-5".into(),
                 messages: json!([{"role": "user", "content": "hi"}]),
                 optional_params: json!({"max_tokens": 16}).as_object().unwrap().clone(),
-                api_key: Some("sk-test"),
+                api_key: Some("sk-test".into()),
-                api_base: Some(api_base),
+                api_base: Some(api_base.into()),
                 custom_llm_provider: None,
                 extra_headers: None,
                 timeout: None,
-            })
+            },
-            .unwrap(),
+            config,
-            std::sync::Arc::new(|_: &str| None),
         )
-        .unwrap()
+        .unwrap();
+        (provider, config, request)
     }
 
     #[rstest]
@@ -247,10 +281,14 @@ mod tests {
             .await;
         let interceptors = RecordingHooks::default();
 
+        let (provider, config, request) = prepared(&upstream.uri());
         execute(
             &Client::plain_for_test(),
             &AuthServices::default(),
-            prepared(&upstream.uri()),
+            config,
+            provider,
+            request,
+            std::sync::Arc::new(|_: &str| None),
             None,
             None,
             &interceptors,
@@ -288,10 +326,14 @@ mod tests {
             .await;
         let interceptors = RecordingHooks::default();
 
+        let (provider, config, request) = prepared(&upstream.uri());
         let error = execute(
             &Client::plain_for_test(),
             &AuthServices::default(),
-            prepared(&upstream.uri()),
+            config,
+            provider,
+            request,
+            std::sync::Arc::new(|_: &str| None),
             None,
             None,
             &interceptors,

litellm-rust/crates/inference-chat/src/lib.rs

@@ -5,14 +5,13 @@ pub use litellm_inference::RouteError as Error;
 mod common_utils;
 pub mod constants;
 pub(crate) mod handler;
-mod prepare;
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
-use prepare::{prepare_provider_request, resolve_request};
 
 use litellm_auth::AuthServices;
+use litellm_inference::provider::resolve_llm_provider;
 use litellm_secrets::source::SecretSource;
 use std::sync::Arc;
-use types::ChatCompletionsRequest;
+use types::{ChatCompletionsCall, ChatCompletionsRequest};
 
 #[derive(Clone)]
 pub struct ChatCompletionsRoute {
@@ -55,7 +54,7 @@ impl ChatCompletionsRoute {
         } = options.into();
         litellm_host::lifecycle::observe_unary(
             observers.clone(),
-            self.run_call(
+            self.run(
                 request.into(),
                 cache_options,
                 interceptors,
@@ -65,30 +64,52 @@ impl ChatCompletionsRoute {
         .await
     }
 
+    #[tracing::instrument(name = "litellm.route", skip_all, fields(
+        route = "chat_completions",
+        model = %call.model,
+        provider,
+        resolved_model,
+        stream = false,
+        outcome
+    ))]
     async fn run(
         &self,
-        request: ChatCompletionsRequest<'_>,
+        call: ChatCompletionsCall,
         cache_options: Option<litellm_cache_response::CachePolicy>,
         interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
         observers: Option<&ObservationSender>,
     ) -> Result<ChatCompletionsResponse, Error> {
-        let resolved = resolve_request(request)?;
+        litellm_inference::diagnostic::unary(async {
-        let snapshot = self
+            let model = call.model.clone();
-            .secrets
+            let custom_llm_provider = call.custom_llm_provider.clone();
-            .resolve(&resolved.config.secret_names())
+            let provider =
-            .await?;
+                resolve_llm_provider(&model, custom_llm_provider.as_deref(), "chat completions")?;
-        let prepared = prepare_provider_request(resolved, snapshot)?;
+            let config = common_utils::chat_completions_provider(provider.provider)
-        litellm_inference::diagnostic::provider(&prepared.model, &prepared.custom_llm_provider);
+                .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?
-        let execute: futures_util::future::BoxFuture<'_, Result<ChatCompletionsResponse, Error>> =
+                .config();
-            Box::pin(handler::execute(
+            let call = common_utils::resolve_request(call, config)?;
+            litellm_inference::diagnostic::provider(
+                provider.model,
+                <&'static str>::from(provider.provider),
+            );
+            let secrets = self.secrets.resolve(&config.secret_names()).await?;
+            let execute: futures_util::future::BoxFuture<
+                '_,
+                Result<ChatCompletionsResponse, Error>,
+            > = Box::pin(handler::execute(
                 &self.http,
                 &self.auth,
-                prepared,
+                config,
+                provider,
+                call,
+                secrets,
                 self.cache.clone(),
                 cache_options,
                 interceptors,
                 observers,
             ));
-        execute.await
+            execute.await
+        })
+        .await
     }
 }

litellm-rust/crates/inference-chat/src/prepare.rs

@@ -1,638 +0,0 @@
-use litellm_auth::SecretValue;
-use litellm_core_utils::settings::Lookup;
-use litellm_http::request::with_default_headers;
-use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
-use litellm_llms_types::formats::chat_completions::ChatMessage;
-use litellm_secrets::source::Secrets;
-use serde_json::Value;
-
-use super::{
-    Error,
-    common_utils::{chat_completions_provider, string_headers},
-};
-use crate::types::{
-    ChatCompletionsRequest, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest,
-};
-use litellm_inference::provider::resolve_llm_provider;
-
-pub(super) struct ResolvedProvider {
-    pub(super) model: String,
-    pub(super) custom_llm_provider: String,
-    pub(super) config: &'static dyn BaseConfig,
-}
-
-pub(super) fn resolve_provider_config<'a>(
-    model: &'a str,
-    custom_llm_provider: Option<&'a str>,
-) -> Result<ResolvedProvider, Error> {
-    let provider_info = resolve_llm_provider(model, custom_llm_provider, "chat completions")?;
-    let config = chat_completions_provider(provider_info.provider)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?
-        .config();
-    Ok(ResolvedProvider {
-        model: provider_info.model.to_string(),
-        custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
-        config,
-    })
-}
-
-pub(super) fn parse_messages(messages: Value) -> Result<Vec<ChatMessage>, Error> {
-    serde_json::from_value(messages).map_err(|err| {
-        Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
-            "chat completions messages",
-            err,
-        ))
-    })
-}
-
-pub(super) fn resolve_request(
-    request: ChatCompletionsRequest<'_>,
-) -> Result<ResolvedChatCompletionsRequest<'_>, Error> {
-    let ResolvedProvider {
-        model,
-        custom_llm_provider,
-        config,
-    } = resolve_provider_config(request.model, request.custom_llm_provider)?;
-    let messages = parse_messages(request.messages)?;
-    if messages.is_empty() {
-        return Err(Error::InvalidRequest(
-            "chat completions requires at least one message".into(),
-        ));
-    }
-    if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) {
-        return Err(Error::Unsupported(reason.0));
-    }
-    Ok(ResolvedChatCompletionsRequest {
-        model,
-        custom_llm_provider,
-        config,
-        messages,
-        optional_params: request.optional_params,
-        api_key: request.api_key,
-        api_base: request.api_base,
-        extra_headers: request.extra_headers,
-        timeout: request.timeout,
-    })
-}
-
-fn validate_environment(
-    request: &ResolvedChatCompletionsRequest<'_>,
-    model: &str,
-    config: &dyn BaseConfig,
-    secrets: &dyn Lookup,
-) -> Result<ValidatedEnvironment, Error> {
-    let env_lookup = |key: &str| secrets.get(key);
-    let forwarded = string_headers(request.extra_headers.clone())?;
-    let validated = config.validate_environment(
-        forwarded,
-        request.api_key,
-        model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    Ok(ValidatedEnvironment {
-        headers: with_default_headers(validated.headers, config.default_headers()),
-        auth: validated.auth,
-    })
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) fn prepare_provider_request(
-    request: ResolvedChatCompletionsRequest<'_>,
-    secrets: Secrets,
-) -> Result<ProviderChatCompletionsRequest, Error> {
-    let environment =
-        validate_environment(&request, &request.model, request.config, secrets.as_ref())?;
-    let model = request.model;
-    let config = request.config;
-    let env_lookup = |key: &str| secrets.get(key);
-    let url = config.get_complete_url(
-        request.api_base,
-        &model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    let transformed =
-        config.transform_request(&model, request.messages, request.optional_params.clone())?;
-
-    Ok(ProviderChatCompletionsRequest {
-        model,
-        custom_llm_provider: request.custom_llm_provider,
-        config,
-        url,
-        body: transformed.body,
-        optional_params: request.optional_params,
-        environment,
-        secrets,
-        timeout: request.timeout,
-        api_key: request.api_key.map(|key| SecretValue::new(key.to_string())),
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use litellm_auth::CredentialPlacement;
-    use litellm_llms::base_llm::auth::{AuthScheme, resolve_auth};
-    use serde_json::{Map, Value, json};
-
-    use super::{prepare_provider_request, resolve_request};
-    use crate::{
-        Error,
-        types::{ChatCompletionsRequest, ProviderChatCompletionsRequest},
-    };
-
-    fn prepare_chat_completions_call(
-        request: ChatCompletionsRequest<'_>,
-    ) -> Result<ProviderChatCompletionsRequest, Error> {
-        prepare_provider_request(
-            resolve_request(request)?,
-            std::sync::Arc::new(|_: &str| None),
-        )
-    }
-
-    /// The headers as they go on the wire, credential applied.
-    fn wire_headers(prepared: &ProviderChatCompletionsRequest) -> Vec<(String, String)> {
-        tokio::runtime::Builder::new_current_thread()
-            .build()
-            .unwrap()
-            .block_on(resolve_auth(
-                &litellm_auth::AuthServices::default(),
-                prepared.environment.clone(),
-                &|_| None,
-            ))
-            .unwrap()
-            .headers
-    }
-
-    fn request<'a>(
-        model: &'a str,
-        provider: Option<&'a str>,
-        messages: Value,
-        optional_params: Value,
-    ) -> ChatCompletionsRequest<'a> {
-        ChatCompletionsRequest {
-            model,
-            messages,
-            optional_params: match optional_params {
-                Value::Object(map) => map,
-                other => panic!("params must be an object, got {other}"),
-            },
-            api_key: Some("sk-test"),
-            api_base: None,
-            custom_llm_provider: provider,
-            extra_headers: None,
-            timeout: None,
-        }
-    }
-
-    fn preparation_error(request: ChatCompletionsRequest<'_>) -> Error {
-        prepare_chat_completions_call(request)
-            .err()
-            .expect("request preparation should fail")
-    }
-
-    #[test]
-    fn resolves_the_provider_from_the_model_prefix() {
-        let prepared = prepare_chat_completions_call(request(
-            "anthropic/claude-sonnet-4-5",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"max_tokens": 16}),
-        ))
-        .expect("prepares");
-        assert_eq!(prepared.model, "claude-sonnet-4-5");
-        assert_eq!(prepared.url, "https://api.anthropic.com/v1/messages");
-        assert_eq!(prepared.body["model"], json!("claude-sonnet-4-5"));
-    }
-
-    #[test]
-    fn strips_an_explicit_provider_prefix_from_the_model() {
-        let prepared = prepare_chat_completions_call(request(
-            "anthropic/claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        ))
-        .expect("prepares");
-        assert_eq!(prepared.model, "claude-sonnet-4-5");
-    }
-
-    #[test]
-    fn adds_the_auth_and_default_headers() {
-        let prepared = prepare_chat_completions_call(request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        ))
-        .expect("prepares");
-        assert!(
-            wire_headers(&prepared).contains(&("x-api-key".to_string(), "sk-test".to_string()))
-        );
-        assert!(
-            wire_headers(&prepared)
-                .contains(&("anthropic-version".to_string(), "2023-06-01".to_string()))
-        );
-        assert!(matches!(
-            prepared.environment.auth,
-            AuthScheme::Credential {
-                placement: CredentialPlacement::Header("x-api-key"),
-                ..
-            }
-        ));
-    }
-
-    #[test]
-    fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
-        // Python builds `{**headers, **anthropic_headers}`, so the deployment's key
-        // overwrites a forwarded one. Honouring the caller's would let whoever sends
-        // the request choose the Anthropic principal it bills to.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "X-Api-Key".to_string(),
-            json!("sk-caller"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .collect();
-        assert_eq!(keys.len(), 1, "got {:?}", headers);
-        assert_eq!(keys[0].1, "sk-test");
-    }
-
-    #[test]
-    fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
-        // Anthropic's `validate_environment` pops `x-api-key` and sets `authorization`
-        // for an OAuth token, so re-adding the key here would put the credential into
-        // a header the host removed on purpose.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([
-            (
-                "Authorization".to_string(),
-                json!("Bearer sk-ant-oat01-token"),
-            ),
-            ("X-Api-Key".to_string(), json!("sk-caller")),
-        ]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        assert!(
-            !wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("x-api-key") && value == "sk-test"),
-            "the resolved key must not be applied over an OAuth bearer, got {:?}",
-            wire_headers(&prepared)
-        );
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-ant-oat01-token")
-        );
-    }
-
-    #[test]
-    fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
-        // Only an OAuth bearer replaces the credential. Python sends the deployment's
-        // `x-api-key` alongside any other forwarded `authorization`, so deferring on
-        // the mere presence of that header would drop the deployment's auth.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([
-            ("Authorization".to_string(), json!("Bearer unrelated")),
-            ("X-Api-Key".to_string(), json!("sk-caller")),
-        ]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .collect();
-        assert_eq!(keys.len(), 1, "got {:?}", headers);
-        assert_eq!(keys[0].1, "sk-test");
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer unrelated"),
-            "the unrelated authorization must survive, got {:?}",
-            wire_headers(&prepared)
-        );
-    }
-
-    #[rstest::rstest]
-    fn rejects_empty_messages_before_resolving_credentials() {
-        let call = ChatCompletionsRequest {
-            api_key: None,
-            ..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
-        };
-        assert!(matches!(preparation_error(call), Error::InvalidRequest(_)));
-    }
-
-    #[test]
-    fn rejects_an_unknown_provider() {
-        assert_eq!(
-            preparation_error(request(
-                "openai/gpt-4o",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({}),
-            )),
-            Error::InvalidProvider("openai".to_string())
-        );
-    }
-
-    #[test]
-    fn rejects_a_model_with_no_resolvable_provider() {
-        assert!(matches!(
-            preparation_error(request(
-                "claude-sonnet-4-5",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({}),
-            )),
-            Error::InvalidProvider(_)
-        ));
-    }
-
-    #[test]
-    fn rejects_an_empty_or_malformed_message_list() {
-        assert_eq!(
-            preparation_error(request(
-                "anthropic/claude-sonnet-4-5",
-                None,
-                json!([]),
-                json!({}),
-            )),
-            Error::InvalidRequest(
-                "chat completions requires at least one message"
-                    .to_string()
-                    .into()
-            )
-        );
-        assert!(matches!(
-            preparation_error(request(
-                "anthropic/claude-sonnet-4-5",
-                None,
-                json!("not a list"),
-                json!({}),
-            )),
-            Error::InvalidRequest(_)
-        ));
-    }
-
-    #[test]
-    fn rejects_non_string_extra_headers() {
-        let mut call = request(
-            "anthropic/claude-sonnet-4-5",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))]));
-        assert_eq!(
-            preparation_error(call),
-            Error::Headers(litellm_http::request::HeaderError {
-                context: "chat completions",
-                name: "x-trace".to_string(),
-                actual: "number",
-            })
-        );
-    }
-
-    #[test]
-    fn prepares_a_bedrock_call_without_resolving_credentials() {
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        );
-        call.api_key = None;
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        assert_eq!(
-            prepared.url,
-            "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
-        );
-        assert!(matches!(
-            &prepared.environment.auth,
-            AuthScheme::AwsSigV4 { region, service: "bedrock", .. } if region == "us-east-1"
-        ));
-        // SigV4 signs the serialized body, so prepare must not have added an
-        // Authorization header; the signer does it over the bytes sent.
-        assert!(
-            !prepared
-                .environment
-                .headers
-                .iter()
-                .any(|(name, _)| name.eq_ignore_ascii_case("authorization"))
-        );
-        assert_eq!(prepared.body["inferenceConfig"], json!({"maxTokens": 16}));
-    }
-
-    #[tokio::test]
-    async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
-        // Python signs only the AWS header set and reattaches the rest, so a header
-        // the caller forwarded rides along without joining the canonical request.
-        // Signing it makes Converse 403 on a deployment that works on Python.
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({
-                "maxTokens": 16,
-                "aws_access_key_id": "AKIDEXAMPLE",
-                "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
-            }),
-        );
-        // A key would resolve to a bearer token and never reach the signer.
-        call.api_key = None;
-        call.extra_headers = Some(Map::from_iter([(
-            "x-request-id".to_string(),
-            json!("abc-123"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let authenticated = resolve_auth(
-            &litellm_auth::AuthServices::default(),
-            prepared.environment,
-            &|_| None,
-        )
-        .await
-        .expect("resolves");
-        let signed = crate::handler::outbound_request(
-            authenticated,
-            prepared.url,
-            &prepared.body,
-            prepared.timeout,
-        )
-        .expect("signs");
-
-        let authorization = signed
-            .header("authorization")
-            .expect("carries an authorization header")
-            .to_string();
-        assert!(
-            authorization.starts_with("AWS4-HMAC-SHA256"),
-            "expected a SigV4 signature, got {authorization}"
-        );
-        assert!(
-            !authorization.contains("x-request-id"),
-            "forwarded header reached SignedHeaders: {authorization}"
-        );
-        // It still goes on the wire, it is just not part of the signature.
-        assert!(
-            signed
-                .headers()
-                .iter()
-                .any(|(name, value)| name == "x-request-id" && value == "abc-123"),
-            "forwarded header was dropped instead of reattached"
-        );
-    }
-
-    #[rstest::rstest]
-    #[case::authorization("Authorization")]
-    #[case::amz_date("x-amz-date")]
-    #[case::security_token("x-amz-security-token")]
-    #[case::date("Date")]
-    #[tokio::test]
-    async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
-        let call = ChatCompletionsRequest {
-            api_key: None,
-            extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
-            ..request(
-                "bedrock/us-east-1/anthropic.claude-v2",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({
-                    "maxTokens": 16,
-                    "aws_access_key_id": "AKIDEXAMPLE",
-                    "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
-                }),
-            )
-        };
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let authenticated = resolve_auth(
-            &litellm_auth::AuthServices::default(),
-            prepared.environment,
-            &|_| None,
-        )
-        .await
-        .expect("resolves");
-        let error = crate::handler::outbound_request(
-            authenticated,
-            prepared.url,
-            &prepared.body,
-            prepared.timeout,
-        )
-        .expect_err("conflicting signing headers must fail");
-        assert!(
-            matches!(error, Error::Unsupported(_)),
-            "{forwarded} returned {error:?}"
-        );
-    }
-
-    #[test]
-    fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
-        // `get_request_headers` assigns `headers["Authorization"]` unconditionally
-        // once a bearer token resolves, so the deployment's identity wins on
-        // Python. Keeping the caller's would authorize and bill the call as a
-        // different principal, and only when the deployment carries `rust: true`.
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "Authorization".to_string(),
-            json!("Bearer caller-supplied"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let authorizations: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("authorization"))
-            .map(|(_, value)| value.as_str())
-            .collect();
-        assert_eq!(
-            authorizations,
-            vec!["Bearer sk-test"],
-            "the deployment token must be the only authorization on the wire"
-        );
-    }
-
-    #[test]
-    fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
-        // The opposite precedence, and deliberate: Anthropic's own transform
-        // honours a forwarded OAuth bearer, so the Bedrock fix above must not be
-        // generalized into a rule that the configured key always wins.
-        //
-        // An OAuth bearer is the whole of that exception. This forwarded a plain
-        // `x-api-key` until round 17, which read as the same claim and was not:
-        // Python overwrites a forwarded `x-api-key` with the deployment's.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "authorization".to_string(),
-            json!("Bearer sk-ant-oat01-forwarded"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .map(|(_, value)| value.as_str())
-            .collect();
-        assert!(keys.is_empty(), "got {:?}", headers);
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-ant-oat01-forwarded")
-        );
-    }
-
-    #[test]
-    fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
-        // The configured bearer identity has its own account and quota boundary,
-        // so a request carrying one must not be signed as whatever principal the
-        // host's AWS credentials resolve to.
-        let prepared = prepare_chat_completions_call(request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        ))
-        .expect("prepares");
-        assert!(matches!(
-            &prepared.environment.auth,
-            AuthScheme::Credential { placement: CredentialPlacement::Bearer, secret }
-                if secret.expose() == "sk-test"
-        ));
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-test"),
-            "prepare did not carry the bearer token"
-        );
-    }
-}

litellm-rust/crates/inference-chat/src/route.rs

@@ -1,4 +1,3 @@
-use litellm_host::observation::ObservationSender;
 use std::convert::Infallible;
 
 use litellm_host::{
@@ -7,10 +6,7 @@ use litellm_host::{
 };
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
 
-use super::{
+use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall};
-    ChatCompletionsRoute, Error,
-    types::{ChatCompletionsCall, ChatCompletionsRequest},
-};
 
 pub struct ChatCompletions;
 
@@ -37,44 +33,12 @@ impl ChatCompletionsRoute {
             call,
             observers,
             move |call, _, interceptors, observers| async move {
-                self.run_call(call, cache_options, &interceptors, observers.as_ref())
+                self.run(call, cache_options, &interceptors, observers.as_ref())
                     .await
                     .map(CallOutput::Complete)
             },
         )
     }
-
-    #[tracing::instrument(name = "litellm.route", skip_all, fields(
-        route = "chat_completions",
-        model = %call.model,
-        provider,
-        resolved_model,
-        stream = false,
-        outcome
-    ))]
-    pub(super) async fn run_call(
-        &self,
-        call: ChatCompletionsCall,
-        cache_options: Option<litellm_cache_response::CachePolicy>,
-        interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<ChatCompletionsResponse, Error> {
-        litellm_inference::diagnostic::unary(async {
-            let request = ChatCompletionsRequest {
-                model: &call.model,
-                messages: call.messages,
-                optional_params: call.optional_params,
-                api_key: call.api_key.as_deref(),
-                api_base: call.api_base.as_deref(),
-                custom_llm_provider: call.custom_llm_provider.as_deref(),
-                extra_headers: call.extra_headers,
-                timeout: call.timeout,
-            };
-            self.run(request, cache_options, interceptors, observers)
-                .await
-        })
-        .await
-    }
 }
 
 impl litellm_inference::caching::Cachable for ChatCompletions {

litellm-rust/crates/inference-chat/src/types.rs

@@ -1,8 +1,5 @@
-use litellm_secrets::source::Secrets;
 use std::time::Duration;
 
-use litellm_auth::SecretValue;
-use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
 use litellm_llms_types::formats::chat_completions::ChatMessage;
 use serde_json::{Map, Value};
 
@@ -49,31 +46,11 @@ impl From<ChatCompletionsRequest<'_>> for ChatCompletionsCall {
     }
 }
 
-pub struct ResolvedChatCompletionsRequest<'a> {
+pub struct ResolvedChatCompletionsRequest {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseConfig,
     pub messages: Vec<ChatMessage>,
     pub optional_params: Map<String, Value>,
-    pub api_key: Option<&'a str>,
+    pub api_key: Option<String>,
-    pub api_base: Option<&'a str>,
+    pub api_base: Option<String>,
     pub extra_headers: Option<Map<String, Value>>,
     pub timeout: Option<Duration>,
 }
-
-pub struct ProviderChatCompletionsRequest {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseConfig,
-    pub url: String,
-    pub body: Value,
-    /// The route's parameters before the provider transformation, reported to the host
-    /// beside the wire request.
-    pub optional_params: Map<String, Value>,
-    /// The forwarded and default headers plus how the call authenticates; the credential
-    /// itself is applied when the request is sent.
-    pub environment: ValidatedEnvironment,
-    pub secrets: Secrets,
-    pub timeout: Option<Duration>,
-    pub api_key: Option<SecretValue>,
-}

litellm-rust/crates/inference-chat/tests/request.rs

@@ -0,0 +1,439 @@
+use std::time::Duration;
+
+use litellm_inference_chat::{Error, types::ChatCompletionsRequest};
+use rstest::rstest;
+use serde_json::{Map, Value, json};
+use wiremock::ResponseTemplate;
+
+mod support;
+use support::*;
+
+const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
+
+fn request<'a>(
+    model: &'a str,
+    provider: Option<&'a str>,
+    messages: Value,
+    optional_params: Value,
+) -> ChatCompletionsRequest<'a> {
+    let Value::Object(optional_params) = optional_params else {
+        panic!("params must be an object");
+    };
+    ChatCompletionsRequest {
+        model,
+        messages,
+        optional_params,
+        api_key: Some("sk-test"),
+        api_base: None,
+        custom_llm_provider: provider,
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(10)),
+    }
+}
+
+async fn complete(
+    request: ChatCompletionsRequest<'_>,
+) -> Result<litellm_llms_types::formats::chat_completions::ChatCompletionsResponse, Error> {
+    chat_completions_route().execute(request, &(), None).await
+}
+
+fn object(value: Value) -> Map<String, Value> {
+    let Value::Object(map) = value else {
+        panic!("expected an object");
+    };
+    map
+}
+
+fn anthropic_response() -> ResponseTemplate {
+    ResponseTemplate::new(200).set_body_raw(ANTHROPIC_MESSAGE, "application/json")
+}
+
+fn bedrock_response() -> ResponseTemplate {
+    json_response(json!({
+        "output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}},
+        "stopReason": "end_turn",
+        "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}
+    }))
+}
+
+#[rstest]
+#[tokio::test]
+async fn resolves_the_provider_from_the_model_prefix() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "anthropic/claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"max_tokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.url.path(), "/v1/messages");
+    assert_eq!(sent.json()["model"], "claude-sonnet-4-5");
+}
+
+#[rstest]
+#[tokio::test]
+async fn strips_an_explicit_provider_prefix_from_the_model() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "anthropic/claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.json()["model"],
+        "claude-sonnet-4-5"
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn adds_the_auth_and_default_headers() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
+    assert_eq!(sent.header("anthropic-version"), Some("2023-06-01"));
+}
+
+#[rstest]
+#[tokio::test]
+async fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({"X-Api-Key": "sk-caller"}))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("x-api-key"),
+        ["sk-test"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({
+            "Authorization": "Bearer sk-ant-oat01-token",
+            "X-Api-Key": "sk-caller"
+        }))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-caller"]);
+    assert_eq!(
+        sent.header_values("authorization"),
+        ["Bearer sk-ant-oat01-token"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({
+            "Authorization": "Bearer unrelated",
+            "X-Api-Key": "sk-caller"
+        }))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
+    assert_eq!(sent.header_values("authorization"), ["Bearer unrelated"]);
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_empty_messages_before_resolving_credentials() {
+    let error = complete(ChatCompletionsRequest {
+        api_key: None,
+        ..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
+    })
+    .await
+    .expect_err("empty messages are rejected");
+    assert!(matches!(error, Error::InvalidRequest(_)));
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_an_unknown_provider() {
+    assert_eq!(
+        complete(request(
+            "openai/gpt-4o",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        ))
+        .await
+        .unwrap_err(),
+        Error::InvalidProvider("openai".into())
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_a_model_with_no_resolvable_provider() {
+    assert!(matches!(
+        complete(request(
+            "claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        ))
+        .await,
+        Err(Error::InvalidProvider(_))
+    ));
+}
+
+#[rstest]
+#[case::empty(json!([]), Some("chat completions requires at least one message"))]
+#[case::malformed(json!("not a list"), None)]
+#[tokio::test]
+async fn rejects_an_empty_or_malformed_message_list(
+    #[case] messages: Value,
+    #[case] expected: Option<&str>,
+) {
+    let error = complete(request(
+        "anthropic/claude-sonnet-4-5",
+        None,
+        messages,
+        json!({}),
+    ))
+    .await
+    .unwrap_err();
+    if let Some(expected) = expected {
+        assert_eq!(error, Error::InvalidRequest(expected.to_string().into()));
+    } else {
+        assert!(matches!(error, Error::InvalidRequest(_)));
+    }
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_non_string_extra_headers() {
+    assert_eq!(
+        complete(ChatCompletionsRequest {
+            extra_headers: Some(Map::from_iter([("x-trace".to_string(), json!(7))])),
+            ..request(
+                "anthropic/claude-sonnet-4-5",
+                None,
+                json!([{"role": "user", "content": "hi"}]),
+                json!({}),
+            )
+        })
+        .await
+        .unwrap_err(),
+        Error::Headers(litellm_http::request::HeaderError {
+            context: "chat completions",
+            name: "x-trace".to_string(),
+            actual: "number",
+        })
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    let mut call = request(
+        "bedrock/us-east-1/anthropic.claude-v2",
+        None,
+        json!([{"role": "user", "content": "hi"}]),
+        json!({
+            "maxTokens": 16,
+            "aws_access_key_id": "AKIDEXAMPLE",
+            "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
+        }),
+    );
+    call.api_key = None;
+    call.api_base = Some(&base);
+    call.extra_headers = Some(Map::from_iter([(
+        "x-request-id".to_string(),
+        json!("abc-123"),
+    )]));
+    complete(call).await.expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    let authorization = sent.header("authorization").expect("signed request");
+    assert!(authorization.starts_with("AWS4-HMAC-SHA256"));
+    assert!(!authorization.contains("x-request-id"));
+    assert_eq!(sent.header("x-request-id"), Some("abc-123"));
+    assert_eq!(sent.url.path(), "/model/anthropic.claude-v2/converse");
+    let body: Value = serde_json::from_slice(&sent.body).unwrap();
+    assert_eq!(body["inferenceConfig"], json!({"maxTokens": 16}));
+    assert_eq!(
+        body["messages"],
+        json!([{"role": "user", "content": [{"text": "hi"}]}])
+    );
+}
+
+#[rstest]
+#[case::authorization("Authorization")]
+#[case::amz_date("x-amz-date")]
+#[case::security_token("x-amz-security-token")]
+#[case::date("Date")]
+#[tokio::test]
+async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
+    let upstream = wiremock::MockServer::start().await;
+    let base = upstream.uri();
+    let call = ChatCompletionsRequest {
+        api_key: None,
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({
+                "maxTokens": 16,
+                "aws_access_key_id": "AKIDEXAMPLE",
+                "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
+            }),
+        )
+    };
+    let error = complete(call)
+        .await
+        .expect_err("conflicting signing headers fail");
+    assert!(matches!(error, Error::Unsupported(_)), "{error:?}");
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(
+            "Authorization".to_string(),
+            json!("Bearer caller-supplied"),
+        )])),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"maxTokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("authorization"),
+        ["Bearer sk-test"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(
+            "authorization".to_string(),
+            json!("Bearer sk-ant-oat01-forwarded"),
+        )])),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert!(sent.header_values("x-api-key").is_empty());
+    assert_eq!(
+        sent.header_values("authorization"),
+        ["Bearer sk-ant-oat01-forwarded"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"maxTokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("authorization"),
+        ["Bearer sk-test"]
+    );
+}

litellm-rust/crates/inference-messages/AGENTS.md

@@ -7,8 +7,8 @@
   - delegate authentication policy, beta selection, payload rewriting, and response interpretation to those adapters
   - keep provider policy out of request preparation and transport handlers
   - calling a concrete provider helper for every provider is still a policy dependency
-- Route types such as `MessagesCall`, prepared requests, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
+- Route types such as `MessagesCall`, provider calls, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
 - The route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks
-- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `src/lib.rs` explicitly sequences cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
+- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `handler::execute` explicitly sequences the provider call build in Python's handler order, cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
 - Preserve the order of validation, normalization, caller-requested parameter removal, and provider transformation when that order affects observable behavior
 - Test provider dispatch, auth precedence, header handling, transformations, and responses through behavior, not source structure

litellm-rust/crates/inference-messages/src/common_utils.rs

@@ -82,17 +82,17 @@ mod tests {
     use litellm_core_utils::get_llm_provider_logic::LlmProviders;
 
     #[rstest]
-    #[case::tencent("tencent")]
+    #[case::tencent(LlmProviders::Tencent, "tencent")]
-    #[case::anthropic("anthropic")]
+    #[case::anthropic(LlmProviders::Anthropic, "anthropic")]
-    #[case::azure_ai("azure_ai")]
+    #[case::azure_ai(LlmProviders::AzureAi, "azure_ai")]
-    #[case::bedrock("bedrock")]
+    #[case::bedrock(LlmProviders::Bedrock, "bedrock")]
-    #[case::deepseek("deepseek")]
+    #[case::deepseek(LlmProviders::Deepseek, "deepseek")]
-    #[case::edenai("edenai")]
+    #[case::edenai(LlmProviders::Edenai, "edenai")]
-    #[case::minimax("minimax")]
+    #[case::minimax(LlmProviders::Minimax, "minimax")]
-    #[case::vertex_ai("vertex_ai")]
+    #[case::vertex_ai(LlmProviders::VertexAi, "vertex_ai")]
-    fn provider_keeps_its_python_name(#[case] name: &str) {
+    #[case::openrouter(LlmProviders::Openrouter, "openrouter")]
-        let provider =
+    fn provider_keeps_its_python_name(#[case] provider: LlmProviders, #[case] name: &str) {
-            messages_provider(name.parse::<LlmProviders>().unwrap(), "claude-sonnet-4-5").unwrap();
+        let provider = messages_provider(provider, "claude-sonnet-4-5").unwrap();
         assert_eq!(provider.as_str(), name);
     }
 

litellm-rust/crates/inference-messages/src/handler.rs

@@ -1,148 +1,271 @@
 use std::time::Duration;
 
 use bytes::Bytes;
-use futures_util::{StreamExt, TryStreamExt, stream::BoxStream};
+use futures_util::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream};
-use litellm_host::interceptors::{Interceptors, ProviderIdentity, RequestContext, WireRequest};
+use litellm_auth::{AuthServices, SecretValue};
-use litellm_http::transport::Error as TransportError;
+use litellm_core_utils::{
+    dot_notation_indexing::delete_nested_value,
+    get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
+};
+use litellm_host::interceptors::{
+    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
+};
+use litellm_http::{request::with_default_headers, transport::Error as TransportError};
 use litellm_llms::base_llm::{
-    auth::{Authenticated, resolve_auth},
+    auth::{Authenticated, ValidatedEnvironment, resolve_auth},
     messages::{
+        context::MessagesTransformContext,
         streaming::{self, ByteStream, StreamDecoder},
         transformation::BaseMessagesConfig,
     },
 };
-use litellm_llms_types::formats::messages::MessagesResponse;
+use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_secrets::source::Secrets;
 use litellm_tracing::ByteChunk;
 use serde_json::Value;
 
 use super::{
-    Error, MessagesCallResponse, MessagesRoute, common_utils::truncate_error_body,
+    Error, MessagesCall, MessagesCallResponse, MessagesRoute,
-    prepare::ProviderMessagesRequest,
+    common_utils::{MessagesProvider, string_headers, truncate_error_body},
+    constants::MESSAGES_TIMEOUT_SECS,
+    route::Messages,
+    types::invalid_request,
 };
-use crate::constants::MESSAGES_TIMEOUT_SECS;
+use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request};
-use litellm_inference::{context::CallContext, outbound::outbound_request};
 
-pub(super) struct ProviderCall {
+struct ProviderCall {
-    pub identity: ProviderIdentity,
+    identity: ProviderIdentity,
-    pub wire: WireRequest,
+    wire: WireRequest,
-    pub(super) provider: super::common_utils::MessagesProvider,
+    provider: MessagesProvider,
     signer: Option<litellm_auth_aws::SigV4Signer>,
     timeout: Option<Duration>,
     stream: bool,
 }
 
 impl ProviderCall {
-    pub fn cacheable(&self) -> bool {
+    fn cacheable(&self) -> bool {
         self.signer.is_none()
     }
 }
 
-impl MessagesRoute {
+pub(super) async fn execute(
-    pub(super) async fn prepare_outbound(
+    route: &MessagesRoute,
-        &self,
+    provider: MessagesProvider,
-        request: ProviderMessagesRequest,
+    model: String,
-        context: &CallContext<'_, impl Interceptors<Error>>,
+    call: MessagesCall,
-    ) -> Result<ProviderCall, Error> {
+    secrets: Secrets,
-        let ProviderMessagesRequest {
+    context: &CallContext<'_, impl Interceptors<Error>>,
-            provider,
+) -> Result<MessagesCallResponse, Error> {
-            url,
+    let request = provider_call(
-            body,
+        &route.auth,
-            environment,
+        provider,
-            timeout,
+        model,
-            api_key,
+        call,
-        } = request;
+        secrets.as_ref(),
-        let request_context = RequestContext {
+        context,
-            model: body.model.clone(),
+    )
-            custom_llm_provider: provider.as_str().to_string(),
+    .boxed()
-            optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
+    .await?;
-            secret_fields: Vec::new(),
+    let cache = CallCache::<Messages>::from_wire(
-            api_key,
+        route.cache.as_ref().filter(|_| request.cacheable()),
-        };
+        context.cache,
-        let authenticated =
+        &request.identity,
-            resolve_auth(&self.auth, environment, &|key| std::env::var(key).ok()).await?;
+        &request.wire,
-        let identity = ProviderIdentity {
+    );
-            model: request_context.model.clone(),
+    let identity = request.identity.clone();
-            provider: request_context.custom_llm_provider.clone(),
+    let (output, source) = match cache.lookup().await {
-        };
+        Some(hit) => hit,
-        let wire = context
+        None => (
-            .interceptors
+            call_provider(&route.http, request, context).await?,
-            .before_provider_request(
+            ResultSource::Provider,
-                WireRequest {
+        ),
-                    url,
+    };
-                    headers: authenticated.headers,
+    let reported = match &output {
-                    body: provider
+        MessagesCallResponse::Complete(response) => provider.config().reported_cost(response),
-                        .config()
+        MessagesCallResponse::Stream { .. } => None,
-                        .wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
+    };
-                },
+    context
-                request_context,
+        .result_ready(ExecutionFacts {
-            )
+            provider: identity,
-            .await?;
+            cost: Cost::settle(&source, reported),
-        let stream = match wire.body.get("stream") {
+            source: source.clone(),
-            None | Some(Value::Null) => false,
-            Some(Value::Bool(stream)) => *stream,
-            Some(value) => {
-                return Err(Error::InvalidRequest(
-                    litellm_llms::ErrorDetail::InvalidValue {
-                        field: "stream",
-                        expected: "a boolean",
-                        actual: value.clone(),
-                    },
-                ));
-            }
-        };
-        Ok(ProviderCall {
-            identity,
-            wire,
-            provider,
-            signer: authenticated.signer,
-            timeout,
-            stream,
         })
-    }
+        .await?;
+    Ok(cache.finish(output, &source).await)
+}
 
-    pub(super) async fn call_provider(
+async fn provider_call(
-        &self,
+    auth: &AuthServices,
-        request: ProviderCall,
+    provider: MessagesProvider,
-        context: &CallContext<'_, impl Interceptors<Error>>,
+    model: String,
-    ) -> Result<MessagesCallResponse, Error> {
+    call: MessagesCall,
-        let ProviderCall {
+    secrets: &(dyn Lookup + Send + Sync),
-            identity,
+    context: &CallContext<'_, impl Interceptors<Error>>,
-            wire,
+) -> Result<ProviderCall, Error> {
-            provider,
+    let MessagesCall {
-            signer,
+        body,
-            timeout,
+        api_key,
-            stream,
+        api_base,
-        } = request;
+        litellm_params,
-        let provider_name = provider.as_str();
+        extra_headers,
-        log_request_body(provider_name, stream, &wire.body);
+        provider_specific_header,
-        let response = send(
+        timeout,
-            &self.http,
+        shaping,
-            Authenticated {
+        ..
-                headers: wire.headers,
+    } = call;
-                signer,
+    let connection_params = litellm_router_types::LitellmParams {
+        api_base: api_base.clone().or(litellm_params.api_base.clone()),
+        ..litellm_params
+    };
+    let config = provider.config();
+    let env_lookup = |key: &str| secrets.get(key);
+
+    let sanitized = config.shape_request(
+        MessagesRequest { model, ..body },
+        shaping.settings.reasoning_auto_summary,
+    )?;
+    let trimmed =
+        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
+
+    let scoped =
+        get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
+    let forwarded = string_headers(Some(
+        extra_headers.into_iter().flatten().chain(scoped).collect(),
+    ))?;
+    let validated = config.validate_environment(
+        forwarded,
+        api_key.as_deref(),
+        &trimmed.model,
+        &trimmed.params,
+        &connection_params,
+        &env_lookup,
+    )?;
+
+    let body = config.transform_anthropic_messages_request(
+        trimmed,
+        &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
+    )?;
+    let environment = ValidatedEnvironment {
+        headers: config.request_headers(
+            with_default_headers(validated.headers, config.default_headers()),
+            &body,
+        ),
+        auth: validated.auth,
+    };
+
+    let url = config.get_complete_url(
+        api_base.as_deref(),
+        &body.model,
+        &connection_params,
+        body.params.stream == Some(true),
+        &env_lookup,
+    )?;
+
+    let request_context = RequestContext {
+        model: body.model.clone(),
+        custom_llm_provider: provider.as_str().to_string(),
+        optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
+        secret_fields: Vec::new(),
+        api_key: api_key.map(SecretValue::new),
+    };
+    let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?;
+    let identity = ProviderIdentity {
+        model: request_context.model.clone(),
+        provider: request_context.custom_llm_provider.clone(),
+    };
+    let wire = context
+        .interceptors
+        .before_provider_request(
+            WireRequest {
+                url,
+                headers: authenticated.headers,
+                body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
             },
-            &wire.url,
+            request_context,
-            &wire.body,
-            timeout,
         )
         .await?;
-        if !response.status().is_success() {
+    let stream = match wire.body.get("stream") {
-            return Err(provider_error(response).await);
+        None | Some(Value::Null) => false,
-        }
+        Some(Value::Bool(stream)) => *stream,
-        let config = provider.config();
+        Some(value) => {
-        if stream {
+            return Err(Error::InvalidRequest(
-            return Ok(streaming_response(
+                litellm_llms::ErrorDetail::InvalidValue {
-                response,
+                    field: "stream",
-                config.stream_decoder(),
+                    expected: "a boolean",
-                provider_name,
+                    actual: value.clone(),
+                },
             ));
         }
-        let text = response.text().await.map_err(network)?;
+    };
-        log_response_body(&text);
+    Ok(ProviderCall {
-        context.response_received(&text).await?;
+        identity,
-        decode_response(config, &identity.model, &text)
+        wire,
-            .map(|message| MessagesCallResponse::Complete(Box::new(message)))
+        provider,
+        signer: authenticated.signer,
+        timeout: timeout.or_else(|| shaping.settings.request_timeout()),
+        stream,
+    })
+}
+
+fn without_additional_drop_params(
+    request: MessagesRequest,
+    paths: &[String],
+) -> Result<MessagesRequest, Error> {
+    if paths.is_empty() {
+        return Ok(request);
     }
+    let params = serde_json::to_value(request.params).map_err(invalid_request)?;
+    let trimmed = paths
+        .iter()
+        .fold(params, |params, path| delete_nested_value(params, path));
+    Ok(MessagesRequest {
+        params: serde_json::from_value(trimmed).map_err(invalid_request)?,
+        ..request
+    })
+}
+
+async fn call_provider(
+    http: &litellm_http::Client,
+    request: ProviderCall,
+    context: &CallContext<'_, impl Interceptors<Error>>,
+) -> Result<MessagesCallResponse, Error> {
+    let ProviderCall {
+        identity,
+        wire,
+        provider,
+        signer,
+        timeout,
+        stream,
+    } = request;
+    let provider_name = provider.as_str();
+    log_request_body(provider_name, stream, &wire.body);
+    let response = send(
+        http,
+        Authenticated {
+            headers: wire.headers,
+            signer,
+        },
+        &wire.url,
+        &wire.body,
+        timeout,
+    )
+    .await?;
+    if !response.status().is_success() {
+        return Err(provider_error(response).await);
+    }
+    let config = provider.config();
+    if stream {
+        return Ok(streaming_response(
+            response,
+            config.stream_decoder(),
+            provider_name,
+        ));
+    }
+    let text = response.text().await.map_err(network)?;
+    log_response_body(&text);
+    context.response_received(&text).await?;
+    decode_response(config, &identity.model, &text)
+        .map(|message| MessagesCallResponse::Complete(Box::new(message)))
 }
 
 fn serialize_failure(err: serde_json::Error) -> Error {
@@ -271,10 +394,482 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) {
 
 #[cfg(test)]
 mod tests {
-    use rstest::rstest;
+    use litellm_llms_types::headers::ProviderSpecificHeaders;
+    use rstest::{fixture, rstest};
+    use serde_json::{Map, Value, json};
     use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
 
     use super::*;
+    use crate::{MessagesSettings, MessagesShaping, common_utils::messages_provider};
+    use litellm_inference::provider::resolve_llm_provider;
+
+    #[fixture]
+    fn shaping() -> MessagesShaping {
+        MessagesShaping::default()
+    }
+
+    fn body(value: Value) -> MessagesRequest {
+        serde_json::from_value(value).unwrap()
+    }
+
+    async fn provider_call_for(call: MessagesCall) -> Result<ProviderCall, Error> {
+        provider_call_with_secrets(call, &|_: &str| None).await
+    }
+
+    async fn provider_call_with_secrets(
+        call: MessagesCall,
+        secrets: &(dyn Lookup + Send + Sync),
+    ) -> Result<ProviderCall, Error> {
+        let provider = resolve_llm_provider(
+            &call.body.model,
+            call.custom_llm_provider.as_deref(),
+            "messages",
+        )?;
+        let config = messages_provider(provider.provider, provider.model)
+            .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?;
+        let model = provider.model.to_owned();
+        let interceptors = ();
+        let context = CallContext::new(&interceptors, litellm_inference::CallOptions::default());
+        provider_call(
+            &AuthServices::default(),
+            config,
+            model,
+            call,
+            secrets,
+            &context,
+        )
+        .await
+    }
+
+    async fn wire_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
+        provider_call_for(MessagesCall {
+            body: body(fields),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://anthropic.test".into()),
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        })
+        .await
+        .map(|call| call.wire.body)
+    }
+
+    #[rstest]
+    #[case::api_key(
+        &[("ANTHROPIC_API_KEY", "sk-secret")],
+        &[("x-api-key", "sk-secret")],
+        "https://api.anthropic.com/v1/messages"
+    )]
+    #[case::auth_token(
+        &[("ANTHROPIC_AUTH_TOKEN", "token")],
+        &[("authorization", "Bearer token")],
+        "https://api.anthropic.com/v1/messages"
+    )]
+    #[case::api_base(
+        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
+        &[("x-api-key", "sk-secret")],
+        "https://gateway.test/v1/messages"
+    )]
+    #[case::sdk_base_url(
+        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
+        &[("x-api-key", "sk-secret")],
+        "https://sdk.test/v1/messages"
+    )]
+    #[tokio::test]
+    async fn credentials_and_base_come_from_the_resolved_secrets(
+        shaping: MessagesShaping,
+        #[case] secrets: &[(&str, &str)],
+        #[case] expected_auth: &[(&str, &str)],
+        #[case] expected_url: &str,
+    ) {
+        let lookup = |name: &str| {
+            secrets
+                .iter()
+                .find(|(key, _)| *key == name)
+                .map(|(_, value)| value.to_string())
+        };
+        let call = provider_call_with_secrets(
+            MessagesCall {
+                body: body(
+                    json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
+                ),
+                api_key: None,
+                api_base: None,
+                custom_llm_provider: Some("anthropic".into()),
+                litellm_params: Default::default(),
+                extra_headers: None,
+                provider_specific_header: None,
+                timeout: None,
+                shaping,
+            },
+            &lookup,
+        )
+        .await
+        .unwrap();
+        let auth: Vec<(&str, &str)> = call
+            .wire
+            .headers
+            .iter()
+            .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
+            .map(|(name, value)| (name.as_str(), value.as_str()))
+            .collect();
+        assert_eq!(
+            (auth.as_slice(), call.wire.url.as_str()),
+            (expected_auth, expected_url)
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn missing_credentials_are_reported_before_the_request_is_transformed(
+        shaping: MessagesShaping,
+    ) {
+        let result = provider_call_for(MessagesCall {
+            body: body(
+                json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}]}),
+            ),
+            api_key: None,
+            api_base: None,
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        })
+        .await;
+        assert!(
+            matches!(
+                result,
+                Err(Error::Auth(litellm_auth::Error::MissingApiKey { .. }))
+            ),
+            "{:?}",
+            result.err()
+        );
+    }
+
+    #[rstest]
+    #[case::nothing_configured(None, None, None)]
+    #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
+    #[case::call_timeout_wins(
+        Some(Duration::from_secs(30)),
+        Some(1800.0),
+        Some(Duration::from_secs(30))
+    )]
+    #[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
+    #[case::unusable_setting_ignored(None, Some(-1.0), None)]
+    #[case::zero_setting_ignored(None, Some(0.0), None)]
+    #[tokio::test]
+    async fn timeout_falls_back_to_the_configured_request_timeout(
+        shaping: MessagesShaping,
+        #[case] call_timeout: Option<Duration>,
+        #[case] request_timeout: Option<f64>,
+        #[case] expected: Option<Duration>,
+    ) {
+        let call = provider_call_for(MessagesCall {
+            body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://anthropic.test".into()),
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: call_timeout,
+            shaping: MessagesShaping {
+                settings: MessagesSettings { request_timeout, ..shaping.settings },
+                ..shaping
+            },
+        })
+        .await
+        .unwrap();
+        assert_eq!(call.timeout, expected);
+    }
+
+    #[rstest]
+    #[case::top_level_and_nested_paths(
+        json!({
+            "max_tokens": 1024,
+            "thinking": {"type": "enabled", "budget_tokens": 2048},
+            "context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
+            "metadata": {"user_id": "u1"},
+            "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
+        }),
+        &["thinking", "context_management", "tools[*].input_examples"],
+        json!({
+            "max_tokens": 1024,
+            "metadata": {"user_id": "u1"},
+            "tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
+        }),
+    )]
+    #[case::no_paths(
+        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[],
+        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
+    )]
+    #[case::model_and_messages_are_never_dropped(
+        json!({"max_tokens": 16}),
+        &["model", "messages", "messages[0].content"],
+        json!({"max_tokens": 16}),
+    )]
+    #[tokio::test]
+    async fn wire_body_drops_configured_paths(
+        shaping: MessagesShaping,
+        #[case] fields: Value,
+        #[case] additional_drop_params: &[&str],
+        #[case] expected_fields: Value,
+    ) {
+        let with_messages = |fields: Value| -> Value {
+            let Value::Object(fields) = fields else {
+                unreachable!()
+            };
+            Value::Object(
+                [
+                    ("model".to_string(), json!("claude-test")),
+                    (
+                        "messages".to_string(),
+                        json!([{"role": "user", "content": "hi"}]),
+                    ),
+                ]
+                .into_iter()
+                .chain(fields)
+                .collect(),
+            )
+        };
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                additional_drop_params: additional_drop_params
+                    .iter()
+                    .map(ToString::to_string)
+                    .collect(),
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert_eq!(
+            wire_body(with_messages(fields), shaping).await,
+            Ok(with_messages(expected_fields))
+        );
+    }
+
+    #[rstest]
+    #[case::model_prefix_picks_the_provider(
+        "azure_ai/claude-test",
+        None,
+        &[("x-priority", "extra"), ("x-scoped", "azure_ai")]
+    )]
+    #[case::explicit_provider(
+        "claude-test",
+        Some("anthropic"),
+        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
+    )]
+    #[case::provider_prefix_on_an_anthropic_model(
+        "anthropic/claude-test",
+        None,
+        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
+    )]
+    #[tokio::test]
+    async fn provider_specific_headers_follow_the_resolved_provider(
+        shaping: MessagesShaping,
+        #[case] model: &str,
+        #[case] custom_llm_provider: Option<&str>,
+        #[case] expected: &[(&str, &str)],
+    ) {
+        let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
+            {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
+            {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
+        ]))
+        .unwrap();
+        let call = provider_call_for(MessagesCall {
+            body: body(
+                json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
+            ),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://resource.services.ai.azure.com".into()),
+            custom_llm_provider: custom_llm_provider.map(Into::into),
+            litellm_params: Default::default(),
+            extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
+            provider_specific_header: Some(configured),
+            timeout: None,
+            shaping,
+        })
+        .await
+        .unwrap();
+        let caller_headers: Vec<(&str, &str)> = call
+            .wire
+            .headers
+            .iter()
+            .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
+            .map(|(name, value)| (name.as_str(), value.as_str()))
+            .collect();
+        assert_eq!(caller_headers, expected);
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn wire_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "anthropic/claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16
+                }),
+                shaping,
+            )
+            .await,
+            Ok(json!({
+                "model": "claude-test",
+                "messages": [{"role": "user", "content": "hi"}],
+                "max_tokens": 16
+            }))
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                reasoning_auto_summary: true,
+                additional_drop_params: vec!["thinking.display".to_string()],
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 4096,
+                    "thinking": {"type": "enabled", "budget_tokens": 2048}
+                }),
+                shaping,
+            )
+            .await,
+            Ok(json!({
+                "model": "claude-test",
+                "messages": [{"role": "user", "content": "hi"}],
+                "max_tokens": 4096,
+                "thinking": {"type": "enabled", "budget_tokens": 2048}
+            }))
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(
+        shaping: MessagesShaping,
+    ) {
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                additional_drop_params: vec!["metadata.user_id".to_string()],
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert!(matches!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16,
+                    "metadata": {"user_id": 123}
+                }),
+                shaping,
+            )
+            .await,
+            Err(Error::InvalidRequest(_))
+        ));
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn wire_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16,
+                    "metadata": {"user_id": 123}
+                }),
+                shaping,
+            )
+            .await,
+            Err(Error::InvalidRequest(
+                litellm_llms::ErrorDetail::InvalidValue {
+                    field: "metadata.user_id",
+                    expected: "a string",
+                    actual: json!(123),
+                }
+            ))
+        );
+    }
+
+    #[rstest]
+    #[case::mantle(
+        "bedrock_mantle/claude-test",
+        "https://bedrock-mantle.us-west-2.api.aws",
+        "/anthropic/v1/messages",
+        "bedrock"
+    )]
+    #[case::platform(
+        "bedrock/claude_platform/claude-test",
+        "https://aws-external-anthropic.us-west-2.api.aws",
+        "/v1/messages",
+        "aws-external-anthropic"
+    )]
+    #[tokio::test]
+    async fn explicit_endpoint_region_is_used_by_the_signer(
+        shaping: MessagesShaping,
+        #[case] model: &str,
+        #[case] base: &str,
+        #[case] path: &str,
+        #[case] service: &str,
+    ) {
+        let call = provider_call_with_secrets(MessagesCall {
+            body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
+            api_key: None,
+            api_base: Some(base.into()),
+            custom_llm_provider: None,
+            litellm_params: crate::litellm_params(Map::from_iter([
+                ("model".into(), json!(model)),
+                ("aws_access_key_id".into(), json!("AKIDEXAMPLE")),
+                ("aws_secret_access_key".into(), json!("secret")),
+            ]))
+            .unwrap(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        }, &|name: &str| match name {
+            "AWS_REGION_NAME" => Some("us-east-1".into()),
+            "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
+            _ => None,
+        })
+        .await
+        .unwrap();
+        assert_eq!(call.wire.url, format!("{base}{path}"));
+        let signed = outbound_request(
+            Authenticated {
+                headers: call.wire.headers,
+                signer: call.signer,
+            },
+            call.wire.url,
+            &call.wire.body,
+            None,
+        )
+        .unwrap();
+        let authorization = signed.header("authorization").unwrap_or_default();
+        assert!(
+            authorization.contains(&format!("/us-west-2/{service}/aws4_request")),
+            "{authorization}"
+        );
+    }
 
     #[rstest]
     #[case::event(

litellm-rust/crates/inference-messages/src/lib.rs

@@ -1,18 +1,19 @@
 mod common_utils;
 mod constants;
 mod handler;
-mod prepare;
 pub mod route;
 mod types;
 
 use futures_util::FutureExt;
 use litellm_auth::AuthServices;
-use litellm_host::interceptors::{Cost, ExecutionFacts, Interceptors, ResultSource};
+use litellm_host::interceptors::Interceptors;
-
+use litellm_inference::context::CallContext;
-use litellm_inference::{caching::CallCache, context::CallContext};
+use litellm_inference::provider::resolve_llm_provider;
 use litellm_secrets::source::SecretSource;
 use std::sync::Arc;
 
+use crate::common_utils::messages_provider;
+
 pub use litellm_inference::RouteError as Error;
 pub use types::{
     MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
@@ -74,39 +75,23 @@ impl MessagesRoute {
         context: CallContext<'_, impl Interceptors<Error>>,
     ) -> Result<MessagesCallResponse, Error> {
         litellm_inference::diagnostic::call(async {
-            let prepared = prepare::prepare(call, self.secrets.as_ref()).await?;
+            let provider = resolve_llm_provider(
+                &call.body.model,
+                call.custom_llm_provider.as_deref(),
+                "messages",
+            )?;
+            let config = messages_provider(provider.provider, provider.model).ok_or_else(|| {
+                Error::InvalidProvider(<&str>::from(provider.provider).to_string())
+            })?;
             litellm_inference::diagnostic::provider(
-                &prepared.body.model,
+                provider.model,
-                prepared.provider.as_str(),
+                <&'static str>::from(provider.provider),
             );
-            let request = self.prepare_outbound(prepared, &context).boxed().await?;
+            let model = provider.model.to_owned();
-            let cache = CallCache::<route::Messages>::from_wire(
+            let secrets = self.secrets.resolve(config.config().secret_names()).await?;
-                self.cache.as_ref().filter(|_| request.cacheable()),
+            handler::execute(self, config, model, call, secrets, &context)
-                context.cache,
+                .boxed()
-                &request.identity,
+                .await
-                &request.wire,
-            );
-            let identity = request.identity.clone();
-            let config = request.provider.config();
-            let (output, source) = match cache.lookup().await {
-                Some(hit) => hit,
-                None => (
-                    self.call_provider(request, &context).await?,
-                    ResultSource::Provider,
-                ),
-            };
-            let reported = match &output {
-                MessagesCallResponse::Complete(response) => config.reported_cost(response),
-                MessagesCallResponse::Stream { .. } => None,
-            };
-            context
-                .result_ready(ExecutionFacts {
-                    provider: identity,
-                    cost: Cost::settle(&source, reported),
-                    source: source.clone(),
-                })
-                .await?;
-            Ok(cache.finish(output, &source).await)
         })
         .await
     }

litellm-rust/crates/inference-messages/src/prepare.rs

@@ -1,554 +0,0 @@
-use std::time::Duration;
-
-use litellm_auth::SecretValue;
-use litellm_core_utils::{
-    dot_notation_indexing::delete_nested_value,
-    get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
-};
-use litellm_http::request::with_default_headers;
-use litellm_llms::base_llm::{
-    auth::ValidatedEnvironment, messages::context::MessagesTransformContext,
-};
-use litellm_llms_types::formats::messages::MessagesRequest;
-use litellm_secrets::source::SecretSource;
-
-use super::{
-    Error, MessagesCall,
-    common_utils::{MessagesProvider, messages_provider, string_headers},
-    types::invalid_request,
-};
-use litellm_inference::provider::resolve_llm_provider;
-
-struct ResolvedProvider {
-    model: String,
-    provider: MessagesProvider,
-}
-
-pub(super) struct ProviderMessagesRequest {
-    pub(super) provider: MessagesProvider,
-    pub(super) url: String,
-    pub(super) body: MessagesRequest,
-    pub(super) environment: ValidatedEnvironment,
-    pub(super) timeout: Option<Duration>,
-    /// The caller's own credential, reported to the host beside the wire request.
-    pub(super) api_key: Option<SecretValue>,
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) async fn prepare(
-    call: MessagesCall,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderMessagesRequest, Error> {
-    let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
-    let secrets = secrets
-        .resolve(resolved.provider.config().secret_names())
-        .await?;
-    prepare_provider_request(call, resolved, secrets.as_ref())
-}
-
-fn resolve_provider(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<ResolvedProvider, Error> {
-    let resolved = resolve_llm_provider(model, custom_llm_provider, "messages")?;
-    let provider = messages_provider(resolved.provider, resolved.model)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(resolved.provider).to_string()))?;
-    Ok(ResolvedProvider {
-        model: resolved.model.to_string(),
-        provider,
-    })
-}
-
-fn prepare_provider_request(
-    call: MessagesCall,
-    resolved: ResolvedProvider,
-    secrets: &dyn Lookup,
-) -> Result<ProviderMessagesRequest, Error> {
-    let ResolvedProvider { model, provider } = resolved;
-    let MessagesCall {
-        body,
-        api_key,
-        api_base,
-        litellm_params,
-        extra_headers,
-        provider_specific_header,
-        timeout,
-        shaping,
-        ..
-    } = call;
-    let connection_params = litellm_router_types::LitellmParams {
-        api_base: api_base.clone().or(litellm_params.api_base.clone()),
-        ..litellm_params
-    };
-    let config = provider.config();
-    let env_lookup = |key: &str| secrets.get(key);
-
-    let sanitized = config.shape_request(
-        MessagesRequest { model, ..body },
-        shaping.settings.reasoning_auto_summary,
-    )?;
-    let trimmed =
-        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
-    let transformed = config.transform_anthropic_messages_request(
-        trimmed,
-        &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
-    )?;
-
-    let scoped =
-        get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
-    let forwarded = string_headers(Some(
-        extra_headers.into_iter().flatten().chain(scoped).collect(),
-    ))?;
-    let forwarded = config.request_headers(
-        with_default_headers(forwarded, config.default_headers()),
-        &transformed,
-    );
-    let environment = config.validate_environment(
-        forwarded,
-        api_key.as_deref(),
-        &transformed.model,
-        &connection_params,
-        &env_lookup,
-    )?;
-
-    let url = config.get_complete_url(
-        api_base.as_deref(),
-        &transformed.model,
-        &connection_params,
-        transformed.params.stream == Some(true),
-        &env_lookup,
-    )?;
-
-    Ok(ProviderMessagesRequest {
-        provider,
-        url,
-        body: transformed,
-        environment,
-        timeout: timeout.or_else(|| shaping.settings.request_timeout()),
-        api_key: api_key.map(SecretValue::new),
-    })
-}
-
-fn without_additional_drop_params(
-    request: MessagesRequest,
-    paths: &[String],
-) -> Result<MessagesRequest, Error> {
-    if paths.is_empty() {
-        return Ok(request);
-    }
-    let params = serde_json::to_value(request.params).map_err(invalid_request)?;
-    let trimmed = paths
-        .iter()
-        .fold(params, |params, path| delete_nested_value(params, path));
-    Ok(MessagesRequest {
-        params: serde_json::from_value(trimmed).map_err(invalid_request)?,
-        ..request
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use litellm_llms::base_llm::auth::resolve_auth;
-    use litellm_llms_types::headers::ProviderSpecificHeaders;
-    use rstest::{fixture, rstest};
-    use serde_json::{Map, Value, json};
-
-    use super::*;
-    use crate::{MessagesSettings, MessagesShaping};
-
-    #[fixture]
-    fn shaping() -> MessagesShaping {
-        MessagesShaping::default()
-    }
-
-    fn body(value: Value) -> MessagesRequest {
-        serde_json::from_value(value).unwrap()
-    }
-
-    fn prepare(call: MessagesCall) -> Result<ProviderMessagesRequest, Error> {
-        prepare_with_secrets(call, &|_: &str| None)
-    }
-
-    fn prepare_with_secrets(
-        call: MessagesCall,
-        secrets: &dyn Lookup,
-    ) -> Result<ProviderMessagesRequest, Error> {
-        let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
-        prepare_provider_request(call, resolved, secrets)
-    }
-
-    /// The headers as they go on the wire, credential applied.
-    fn wire_headers(prepared: &ProviderMessagesRequest) -> Vec<(String, String)> {
-        tokio::runtime::Builder::new_current_thread()
-            .build()
-            .unwrap()
-            .block_on(resolve_auth(
-                &litellm_auth::AuthServices::default(),
-                prepared.environment.clone(),
-                &|_| None,
-            ))
-            .unwrap()
-            .headers
-    }
-
-    #[rstest]
-    #[case::api_key(
-        &[("ANTHROPIC_API_KEY", "sk-secret")],
-        &[("x-api-key", "sk-secret")],
-        "https://api.anthropic.com/v1/messages"
-    )]
-    #[case::auth_token(
-        &[("ANTHROPIC_AUTH_TOKEN", "token")],
-        &[("authorization", "Bearer token")],
-        "https://api.anthropic.com/v1/messages"
-    )]
-    #[case::api_base(
-        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
-        &[("x-api-key", "sk-secret")],
-        "https://gateway.test/v1/messages"
-    )]
-    #[case::sdk_base_url(
-        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
-        &[("x-api-key", "sk-secret")],
-        "https://sdk.test/v1/messages"
-    )]
-    fn credentials_and_base_come_from_the_resolved_secrets(
-        shaping: MessagesShaping,
-        #[case] secrets: &[(&str, &str)],
-        #[case] expected_auth: &[(&str, &str)],
-        #[case] expected_url: &str,
-    ) {
-        let lookup = |name: &str| {
-            secrets
-                .iter()
-                .find(|(key, _)| *key == name)
-                .map(|(_, value)| value.to_string())
-        };
-        let prepared = prepare_with_secrets(
-            MessagesCall {
-                body: body(
-                    json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
-                ),
-                api_key: None,
-                api_base: None,
-                custom_llm_provider: Some("anthropic".into()),
-                litellm_params: Default::default(),
-                extra_headers: None,
-                provider_specific_header: None,
-                timeout: None,
-                shaping,
-            },
-            &lookup,
-        )
-        .unwrap();
-        let headers = wire_headers(&prepared);
-        let auth: Vec<(&str, &str)> = headers
-            .iter()
-            .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
-            .map(|(name, value)| (name.as_str(), value.as_str()))
-            .collect();
-        assert_eq!(
-            (auth.as_slice(), prepared.url.as_str()),
-            (expected_auth, expected_url)
-        );
-    }
-
-    #[rstest]
-    #[case::nothing_configured(None, None, None)]
-    #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
-    #[case::call_timeout_wins(
-        Some(Duration::from_secs(30)),
-        Some(1800.0),
-        Some(Duration::from_secs(30))
-    )]
-    #[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
-    #[case::unusable_setting_ignored(None, Some(-1.0), None)]
-    #[case::zero_setting_ignored(None, Some(0.0), None)]
-    fn timeout_falls_back_to_the_configured_request_timeout(
-        shaping: MessagesShaping,
-        #[case] call_timeout: Option<Duration>,
-        #[case] request_timeout: Option<f64>,
-        #[case] expected: Option<Duration>,
-    ) {
-        let prepared = prepare(MessagesCall {
-            body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://anthropic.test".into()),
-            custom_llm_provider: Some("anthropic".into()),
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: call_timeout,
-            shaping: MessagesShaping {
-                settings: MessagesSettings { request_timeout, ..shaping.settings },
-                ..shaping
-            },
-        })
-        .unwrap();
-        assert_eq!(prepared.timeout, expected);
-    }
-
-    fn prepared_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
-        prepare(MessagesCall {
-            body: body(fields),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://anthropic.test".into()),
-            custom_llm_provider: Some("anthropic".into()),
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: None,
-            shaping,
-        })
-        .map(|prepared| serde_json::to_value(prepared.body).unwrap())
-    }
-
-    #[rstest]
-    #[case::top_level_and_nested_paths(
-        json!({
-            "max_tokens": 1024,
-            "thinking": {"type": "enabled", "budget_tokens": 2048},
-            "context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
-            "metadata": {"user_id": "u1"},
-            "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
-        }),
-        &["thinking", "context_management", "tools[*].input_examples"],
-        json!({
-            "max_tokens": 1024,
-            "metadata": {"user_id": "u1"},
-            "tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
-        }),
-    )]
-    #[case::no_paths(
-        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
-        &[],
-        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
-    )]
-    #[case::model_and_messages_are_never_dropped(
-        json!({"max_tokens": 16}),
-        &["model", "messages", "messages[0].content"],
-        json!({"max_tokens": 16}),
-    )]
-    fn prepared_body_drops_configured_paths(
-        shaping: MessagesShaping,
-        #[case] fields: Value,
-        #[case] additional_drop_params: &[&str],
-        #[case] expected_fields: Value,
-    ) {
-        let with_messages = |fields: Value| -> Value {
-            let Value::Object(fields) = fields else {
-                unreachable!()
-            };
-            Value::Object(
-                [
-                    ("model".to_string(), json!("claude-test")),
-                    (
-                        "messages".to_string(),
-                        json!([{"role": "user", "content": "hi"}]),
-                    ),
-                ]
-                .into_iter()
-                .chain(fields)
-                .collect(),
-            )
-        };
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                additional_drop_params: additional_drop_params
-                    .iter()
-                    .map(ToString::to_string)
-                    .collect(),
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert_eq!(
-            prepared_body(with_messages(fields), shaping),
-            Ok(with_messages(expected_fields))
-        );
-    }
-
-    #[rstest]
-    #[case::model_prefix_picks_the_provider(
-        "azure_ai/claude-test",
-        None,
-        &[("x-priority", "extra"), ("x-scoped", "azure_ai")]
-    )]
-    #[case::explicit_provider(
-        "claude-test",
-        Some("anthropic"),
-        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
-    )]
-    #[case::provider_prefix_on_an_anthropic_model(
-        "anthropic/claude-test",
-        None,
-        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
-    )]
-    fn provider_specific_headers_follow_the_resolved_provider(
-        shaping: MessagesShaping,
-        #[case] model: &str,
-        #[case] custom_llm_provider: Option<&str>,
-        #[case] expected: &[(&str, &str)],
-    ) {
-        let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
-            {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
-            {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
-        ]))
-        .unwrap();
-        let prepared = prepare(MessagesCall {
-            body: body(
-                json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
-            ),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://resource.services.ai.azure.com".into()),
-            custom_llm_provider: custom_llm_provider.map(Into::into),
-            litellm_params: Default::default(),
-            extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
-            provider_specific_header: Some(configured),
-            timeout: None,
-            shaping,
-        })
-        .unwrap();
-        let caller_headers: Vec<(&str, &str)> = prepared
-            .environment
-            .headers
-            .iter()
-            .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
-            .map(|(name, value)| (name.as_str(), value.as_str()))
-            .collect();
-        assert_eq!(caller_headers, expected);
-    }
-
-    #[rstest]
-    fn prepared_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "anthropic/claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16
-                }),
-                shaping,
-            ),
-            Ok(json!({
-                "model": "claude-test",
-                "messages": [{"role": "user", "content": "hi"}],
-                "max_tokens": 16
-            }))
-        );
-    }
-
-    #[rstest]
-    fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                reasoning_auto_summary: true,
-                additional_drop_params: vec!["thinking.display".to_string()],
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 4096,
-                    "thinking": {"type": "enabled", "budget_tokens": 2048}
-                }),
-                shaping,
-            ),
-            Ok(json!({
-                "model": "claude-test",
-                "messages": [{"role": "user", "content": "hi"}],
-                "max_tokens": 4096,
-                "thinking": {"type": "enabled", "budget_tokens": 2048}
-            }))
-        );
-    }
-
-    #[rstest]
-    fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(shaping: MessagesShaping) {
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                additional_drop_params: vec!["metadata.user_id".to_string()],
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert!(matches!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16,
-                    "metadata": {"user_id": 123}
-                }),
-                shaping,
-            ),
-            Err(Error::InvalidRequest(_))
-        ));
-    }
-
-    #[rstest]
-    fn prepared_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16,
-                    "metadata": {"user_id": 123}
-                }),
-                shaping,
-            ),
-            Err(Error::InvalidRequest(
-                litellm_llms::ErrorDetail::InvalidValue {
-                    field: "metadata.user_id",
-                    expected: "a string",
-                    actual: json!(123),
-                }
-            ))
-        );
-    }
-    #[rstest]
-    #[case::mantle(
-        "bedrock_mantle/claude-test",
-        "https://bedrock-mantle.us-west-2.api.aws",
-        "/anthropic/v1/messages",
-        "bedrock"
-    )]
-    #[case::platform(
-        "bedrock/claude_platform/claude-test",
-        "https://aws-external-anthropic.us-west-2.api.aws",
-        "/v1/messages",
-        "aws-external-anthropic"
-    )]
-    fn explicit_endpoint_region_is_used_by_the_signer(
-        shaping: MessagesShaping,
-        #[case] model: &str,
-        #[case] base: &str,
-        #[case] path: &str,
-        #[case] service: &str,
-    ) {
-        let prepared = prepare_with_secrets(MessagesCall {
-            body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
-            api_key: None,
-            api_base: Some(base.into()),
-            custom_llm_provider: None,
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: None,
-            shaping,
-        }, &|name: &str| match name {
-            "AWS_REGION_NAME" => Some("us-east-1".into()),
-            "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
-            _ => None,
-        }).unwrap();
-        assert_eq!(prepared.url, format!("{base}{path}"));
-        assert!(
-            matches!(prepared.environment.auth, litellm_llms::base_llm::auth::AuthScheme::AwsSigV4 { region, service: actual, .. } if region == "us-west-2" && actual == service)
-        );
-    }
-}

litellm-rust/crates/inference-messages/tests/messages/request.rs

@@ -343,6 +343,77 @@ async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
     assert_eq!(sent, expected);
 }
 
+#[rstest]
+#[case::anthropic("anthropic", &[AnthropicBeta::DangerousToolUse20260903])]
+#[case::deepseek("deepseek", &[])]
+#[case::minimax("minimax", &[])]
+#[tokio::test]
+async fn safeguards_add_the_dangerous_tool_use_beta_only_where_the_beta_map_has_a_column(
+    call: MessagesCall,
+    #[case] provider: &str,
+    #[case] derived: &[AnthropicBeta],
+) {
+    let upstream = upstream([message_response()]).await;
+    let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
+
+    run_message(with_fields(
+        MessagesCall {
+            custom_llm_provider: Some(provider.into()),
+            api_key: Some("sk".into()),
+            api_base: Some(upstream.uri()),
+            extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
+            ..call
+        },
+        json!({"safeguards": safeguards}),
+    ))
+    .await;
+
+    let request = only_request(&upstream).await;
+    let expected: BetaSet = derived
+        .iter()
+        .cloned()
+        .chain([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
+        .collect();
+    assert_eq!(sent_betas(&request), expected);
+    assert_eq!(
+        request.body_json::<Value>().unwrap()["safeguards"],
+        safeguards
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn null_safeguards_are_dropped_instead_of_sent_without_their_beta(
+    call: MessagesCall,
+    #[values("anthropic", "azure_ai", "bedrock_mantle", "deepseek", "minimax")] provider: &str,
+) {
+    let upstream = upstream([message_response()]).await;
+
+    run_message(with_fields(
+        MessagesCall {
+            custom_llm_provider: Some(provider.into()),
+            api_key: Some("sk".into()),
+            api_base: Some(upstream.uri()),
+            extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
+            ..call
+        },
+        json!({"safeguards": null}),
+    ))
+    .await;
+
+    let request = only_request(&upstream).await;
+    assert_eq!(
+        sent_betas(&request),
+        BetaSet::from_iter([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
+    );
+    assert!(
+        !request
+            .body_json::<Map<String, Value>>()
+            .unwrap()
+            .contains_key("safeguards")
+    );
+}
+
 #[rstest]
 #[tokio::test]
 async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: MessagesCall) {

litellm-rust/crates/inference-ocr/src/arguments.rs

@@ -44,8 +44,8 @@ pub fn consumed_optional_param_names(
     model: &str,
     custom_llm_provider: Option<&str>,
 ) -> Result<Vec<&'static str>, Error> {
-    let (model, config) = resolve_provider_config(model, custom_llm_provider)?;
+    let (provider, config) = resolve_provider_config(model, custom_llm_provider)?;
-    let provider_fields = config.get_supported_ocr_params(&model);
+    let provider_fields = config.get_supported_ocr_params(provider.model);
     let auth_fields: &[&str] = match config {
         OcrConfigKind::AwsTextract | OcrConfigKind::AwsTextractAnalyze => AWS_AUTH_OPTION_FIELDS,
         OcrConfigKind::AzureAi

litellm-rust/crates/inference-ocr/src/client.rs

@@ -1,81 +0,0 @@
-use litellm_host::observation::ObservationSender;
-use std::sync::Arc;
-
-use litellm_host::interceptors::Interceptors;
-use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
-use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
-
-use super::{
-    handler::perform_ocr_request,
-    types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest},
-};
-
-#[derive(Clone)]
-pub struct OcrRoute {
-    client: OcrClient,
-}
-
-impl OcrRoute {
-    pub fn new(client: OcrClient) -> Self {
-        Self { client }
-    }
-
-    pub async fn execute(
-        &self,
-        request: LiteLLMOcrRequest,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<ObservationSender>,
-    ) -> Result<LiteLLMOcrResponse, Error> {
-        litellm_host::lifecycle::observe_unary(
-            observers.clone(),
-            self.run(request, interceptors, observers.as_ref()),
-        )
-        .await
-    }
-
-    #[tracing::instrument(name = "litellm.route", skip_all, fields(
-        route = "ocr",
-        model = %request.model,
-        resolved_model = %request.model,
-        provider = <&str>::from(request.config.provider()),
-        stream = false,
-        outcome
-    ))]
-    pub(super) async fn run(
-        &self,
-        request: LiteLLMOcrRequest,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<LiteLLMOcrResponse, Error> {
-        litellm_inference::diagnostic::unary(async {
-            let caller_document = matches!(&request.document, OcrDocumentInput::Document(_));
-            let prepared = prepare_request_document(request).await?;
-            let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
-                Box::pin(perform_ocr_request(
-                    &self.client,
-                    prepared,
-                    interceptors,
-                    caller_document,
-                    observers,
-                ));
-            execute.await
-        })
-        .await
-    }
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-async fn prepare_request_document(
-    request: LiteLLMOcrRequest<OcrDocumentInput>,
-) -> Result<ResolvedOcrRequest, Error> {
-    if let OcrDocumentInput::Document(_) = &request.document {
-        return request.map_document(super::document::prepare_document);
-    }
-    let logger = litellm_tracing::Logger::current();
-    let span = tracing::Span::current();
-    tokio::task::spawn_blocking(move || {
-        logger.scope(|| span.in_scope(|| request.map_document(super::document::prepare_document)))
-    })
-    .await
-    .map_err(|error| Error::DocumentTask(Arc::new(error)))?
-}

litellm-rust/crates/inference-ocr/src/document.rs

@@ -1,10 +1,10 @@
-use std::{collections::BTreeMap as Map, io::Read, path::Path};
+use std::{collections::BTreeMap as Map, io::Read, path::Path, sync::Arc};
 
 use base64::{Engine, engine::general_purpose::STANDARD};
 use litellm_llms::base_llm::ocr::{error::Error, transformation::OCR_INLINE_MAX_BYTES};
 use litellm_llms_types::formats::ocr::OcrDocument;
 
-use crate::types::OcrDocumentInput;
+use crate::types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest};
 
 pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
     match input {
@@ -24,6 +24,21 @@ pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
     }
 }
 
+pub(crate) async fn resolve_document(
+    request: LiteLLMOcrRequest<OcrDocumentInput>,
+) -> Result<ResolvedOcrRequest, Error> {
+    if let OcrDocumentInput::Document(_) = &request.document {
+        return request.map_document(prepare_document);
+    }
+    let logger = litellm_tracing::Logger::current();
+    let span = tracing::Span::current();
+    tokio::task::spawn_blocking(move || {
+        logger.scope(|| span.in_scope(|| request.map_document(prepare_document)))
+    })
+    .await
+    .map_err(|error| Error::DocumentTask(Arc::new(error)))?
+}
+
 pub fn read_path_document(path: &Path, mime_type: Option<&str>) -> Result<OcrDocument, Error> {
     let mut bytes = Vec::new();
     std::fs::File::open(path)

litellm-rust/crates/inference-ocr/src/handler.rs

@@ -4,29 +4,45 @@ use litellm_host::{lifecycle::ExecutionEvent, observation::ObservationSender};
 use litellm_llms::base_llm::ocr::{
     error::Error,
     handler::{CallHooks, OcrClient},
-    transformation::PreparedOcrRequest,
+    transformation::{OcrConnection, PreparedOcrRequest},
 };
 use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
-use super::{arguments::is_secret_param, prepare::prepare_request, provider_config::OcrConfigKind};
+use super::{arguments::is_secret_param, provider_config::OcrConfigKind};
-use crate::types::ResolvedOcrRequest;
+use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
 
-pub(crate) async fn perform_ocr_request(
+pub(crate) async fn execute(
     client: &OcrClient,
-    request: ResolvedOcrRequest,
+    config: OcrConfigKind,
+    call: ResolvedOcrRequest,
+    secrets: Secrets,
     host: &impl Interceptors<Error>,
     caller_document: bool,
     observers: Option<&ObservationSender>,
 ) -> Result<LiteLLMOcrResponse, Error> {
-    request.response_format()?;
+    call.response_format()?;
-    let config = request.config;
+    let LiteLLMOcrRequest {
-    let secrets = client
+        model,
-        .secret_source()
+        document,
-        .resolve(&config.secret_names())
+        credentials,
-        .await
+        transport,
-        .map_err(|error| Error::Secret(std::sync::Arc::new(error)))?;
+        optional_params,
-    let request = prepare_request(request, caller_document, client, secrets);
+        input_sources,
+        azure_ad_token_provider,
+        ..
+    } = call;
+    let resolved = config.resolve_credentials(credentials, &secrets);
+    let request = PreparedOcrRequest {
+        model,
+        document,
+        connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
+        caller_document,
+        optional_params,
+        input_sources,
+        azure_ad_token_provider,
+    };
     let interceptors = OcrCallHooks::new(host, &request, config, observers);
     config.ocr(client, &request, &interceptors).await
 }
@@ -86,3 +102,257 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
         Box::pin(self.interceptors.after_provider_response(raw))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
+    use litellm_llms_types::formats::ocr::OcrResponseFormat;
+    use rstest::rstest;
+    use serde_json::{Value, json};
+    use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
+
+    use super::*;
+    use crate::{
+        OcrRoute,
+        types::LiteLLMOcrRequest,
+        wire::{OcrWireRequest, decode_request},
+    };
+
+    fn client() -> OcrClient {
+        OcrClient::for_test(
+            litellm_http::Client::plain_for_test(),
+            litellm_http::Client::no_redirect_for_test(),
+        )
+        .with_secrets(litellm_inference_testing::no_secrets())
+    }
+
+    fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
+        decode_request(OcrWireRequest {
+            model: model.into(),
+            document,
+            api_key: Some(litellm_auth::SecretValue::new("test-key")),
+            api_base: Some(base.into()),
+            custom_llm_provider: None,
+            extra_headers: None,
+            optional_params: options.as_object().unwrap().clone(),
+            input_sources: Default::default(),
+            timeout_seconds: Some(2.0),
+        })
+        .unwrap()
+    }
+
+    async fn run_request(request: LiteLLMOcrRequest) -> Result<LiteLLMOcrResponse, Error> {
+        OcrRoute::new(client()).execute(request, &(), None).await
+    }
+
+    async fn upstream(request_count: u64, response: Value) -> MockServer {
+        let server = MockServer::start().await;
+        Mock::given(any())
+            .respond_with(ResponseTemplate::new(200).set_body_json(response))
+            .up_to_n_times(request_count)
+            .mount(&server)
+            .await;
+        server
+    }
+
+    fn image(url: &str) -> Value {
+        json!({"type": "image_url", "image_url": url})
+    }
+
+    fn body(request: &wiremock::Request) -> Value {
+        serde_json::from_slice(&request.body).unwrap()
+    }
+
+    fn assert_mistral_request(request: &wiremock::Request) {
+        assert_eq!(
+            request
+                .headers
+                .get("authorization")
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "Bearer test-key"
+        );
+        assert_eq!(
+            request
+                .headers
+                .get("content-type")
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "application/json"
+        );
+        assert_eq!(
+            body(request),
+            json!({
+                "model": "mistral-ocr-maas",
+                "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
+                "pages": [0, 2],
+                "include_image_base64": true,
+                "unknown": "preserved"
+            })
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
+        let upstream = upstream(
+            1,
+            json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
+        )
+        .await;
+        let request = request(
+            "cohere/parse",
+            &upstream.uri(),
+            image("https://example.com/original.png"),
+            json!({
+                "output_format": "markdown", "timeout": 30,
+                "extra_body": {
+                    "output_format": {"future": true},
+                    "document": {"type": "image_url", "image_url": "https://example.com/a.png",
+                        "provider_options": {"nested": [false, 0, null]}}
+                }
+            }),
+        );
+
+        run_request(request).await.unwrap();
+
+        let [request] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        assert_eq!(
+            body(&request),
+            json!({
+                "model": "parse", "output_format": {"future": true},
+                "document": {"type": "image_url", "image_url": "https://example.com/a.png",
+                    "provider_options": {"nested": [false, 0, null]}}
+            })
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn explicit_null_options_use_defaults_before_http() {
+        let upstream = upstream(
+            1,
+            json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
+        )
+        .await;
+        let request = request(
+            "cohere/parse",
+            &upstream.uri(),
+            image("https://example.com/a.png"),
+            json!({"output_format": null, "req_format": null}),
+        );
+        assert_eq!(
+            request.response_format().unwrap(),
+            OcrResponseFormat::Litellm
+        );
+
+        run_request(request).await.unwrap();
+
+        let [request] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        let body = body(&request);
+        assert_eq!(body["output_format"], "markdown");
+        assert!(body.get("req_format").is_none());
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
+        let upstream = upstream(
+            2,
+            json!({
+                "pages": [{"index": 0, "markdown": "hello"}],
+                "extra": "preserved"
+            }),
+        )
+        .await;
+        let options = json!({
+            "pages": [0, 2],
+            "include_image_base64": true,
+            "vertex_project": "project-1",
+            "vertex_location": "us-central1",
+            "unknown": "preserved"
+        });
+        let document =
+            json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
+        let direct_call = request(
+            "mistral/mistral-ocr-maas",
+            &upstream.uri(),
+            document.clone(),
+            options.clone(),
+        );
+        let vertex_call = request(
+            "vertex_ai/mistral-ocr-maas",
+            &upstream.uri(),
+            document,
+            options,
+        );
+        assert_eq!(direct_call.transport.timeout, Some(Duration::from_secs(2)));
+        assert_eq!(vertex_call.transport.timeout, Some(Duration::from_secs(2)));
+        let direct_response = run_request(direct_call).await.unwrap().into_json();
+        let vertex_response = run_request(vertex_call).await.unwrap().into_json();
+
+        let [direct_http, vertex_http] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        assert_eq!(direct_http.url.path(), "/v1/ocr");
+        assert_eq!(
+            vertex_http.url.path(),
+            "/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
+        );
+        assert_mistral_request(&direct_http);
+        assert_mistral_request(&vertex_http);
+        assert_eq!(direct_response, vertex_response);
+        assert_eq!(direct_response["model"], "mistral-ocr-maas");
+        assert_eq!(direct_response["object"], "ocr");
+        assert_eq!(direct_response["extra"], "preserved");
+    }
+
+    #[derive(serde::Deserialize)]
+    struct KnownParams {
+        pages: Option<Vec<i64>>,
+    }
+
+    #[rstest]
+    fn parsed_provider_params_separates_known_and_extra_params() {
+        let arguments: CallArguments = serde_json::from_value(json!({
+            "pages": [0, 2],
+            "future_ocr_option": true,
+            "extra_body": {"provider_option": "value"}
+        }))
+        .unwrap();
+        let known: KnownParams = parse_options(&arguments).unwrap();
+        assert_eq!(known.pages, Some(vec![0, 2]));
+        assert_eq!(arguments["future_ocr_option"], true);
+        assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
+        assert_eq!(
+            arguments
+                .iter()
+                .filter(|(name, _)| name.as_str() != "pages")
+                .count(),
+            2
+        );
+        assert_eq!(
+            compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
+            json!({
+                "pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
+            })
+        );
+    }
+}

litellm-rust/crates/inference-ocr/src/lib.rs

@@ -1,10 +1,82 @@
 pub mod arguments;
-mod client;
-pub use client::OcrRoute;
 pub mod document;
 pub(crate) mod handler;
-pub(crate) mod prepare;
 pub mod provider_config;
 pub mod route;
 pub mod types;
 pub mod wire;
+
+use std::sync::Arc;
+
+use litellm_host::interceptors::Interceptors;
+use litellm_host::observation::ObservationSender;
+use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
+use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
+
+use crate::{
+    handler::execute,
+    types::{LiteLLMOcrRequest, OcrDocumentInput},
+};
+
+#[derive(Clone)]
+pub struct OcrRoute {
+    client: OcrClient,
+}
+
+impl OcrRoute {
+    pub fn new(client: OcrClient) -> Self {
+        Self { client }
+    }
+
+    pub async fn execute(
+        &self,
+        request: LiteLLMOcrRequest,
+        interceptors: &impl Interceptors<Error>,
+        observers: Option<ObservationSender>,
+    ) -> Result<LiteLLMOcrResponse, Error> {
+        litellm_host::lifecycle::observe_unary(
+            observers.clone(),
+            self.run(request, interceptors, observers.as_ref()),
+        )
+        .await
+    }
+
+    #[tracing::instrument(name = "litellm.route", skip_all, fields(
+        route = "ocr",
+        model = %call.model,
+        resolved_model = %call.model,
+        provider = <&str>::from(call.config.provider()),
+        stream = false,
+        outcome
+    ))]
+    async fn run(
+        &self,
+        call: LiteLLMOcrRequest,
+        interceptors: &impl Interceptors<Error>,
+        observers: Option<&ObservationSender>,
+    ) -> Result<LiteLLMOcrResponse, Error> {
+        litellm_inference::diagnostic::unary(async {
+            let caller_document = matches!(&call.document, OcrDocumentInput::Document(_));
+            let call = document::resolve_document(call).await?;
+            let config = call.config;
+            let secrets = self
+                .client
+                .secret_source()
+                .resolve(&config.secret_names())
+                .await
+                .map_err(|error| Error::Secret(Arc::new(error)))?;
+            let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
+                Box::pin(execute(
+                    &self.client,
+                    config,
+                    call,
+                    secrets,
+                    interceptors,
+                    caller_document,
+                    observers,
+                ));
+            execute.await
+        })
+        .await
+    }
+}

litellm-rust/crates/inference-ocr/src/prepare.rs

@@ -1,309 +0,0 @@
-use litellm_auth::{InputSource, SecretValue, Sourced};
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_llms::base_llm::ocr::{
-    handler::OcrClient,
-    transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
-};
-use litellm_secrets::source::Secrets;
-
-use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
-
-pub(crate) fn prepare_request(
-    request: ResolvedOcrRequest,
-    caller_document: bool,
-    client: &OcrClient,
-    secrets: Secrets,
-) -> PreparedOcrRequest {
-    let credentials = request.credentials.clone();
-    let (preferred_api_key_env, api_base_env) = match request.config.provider() {
-        LlmProviders::Mistral => (
-            Some("MISTRAL_AZURE_API_KEY"),
-            Some("MISTRAL_AZURE_API_BASE"),
-        ),
-        LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
-        _ => (None, None),
-    };
-    let secret = |name: &str| secrets.truthy(name);
-    let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
-        credentials.api_key.clone().or_else(|| {
-            preferred_api_key_env
-                .into_iter()
-                .chain(request.config.get_api_key_env_var())
-                .find_map(secret)
-                .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
-        })
-    });
-    let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
-        credentials.api_base.clone().or_else(|| {
-            api_base_env
-                .and_then(secret)
-                .map(|value| Sourced::new(value, InputSource::Environment))
-        })
-    });
-    let resolved = request
-        .config
-        .resolve_connection_params(OcrCredentialInputs {
-            dynamic_api_key,
-            dynamic_api_base,
-            ..credentials
-        });
-    let LiteLLMOcrRequest {
-        model,
-        document,
-        transport,
-        optional_params,
-        input_sources,
-        azure_ad_token_provider,
-        ..
-    } = request;
-    PreparedOcrRequest {
-        model,
-        document,
-        connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
-        caller_document,
-        optional_params,
-        input_sources,
-        azure_ad_token_provider,
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::time::Duration;
-
-    use futures_util::future::BoxFuture;
-    use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
-    use litellm_host::interceptors::WireRequest;
-    use litellm_llms::{
-        base_llm::ocr::{
-            error::Error,
-            handler::{CallHooks, OcrClient},
-            transformation::BaseOcrConfig,
-        },
-        cohere::ocr::transformation::CohereParseConfig,
-        mistral::ocr::transformation::MistralOcrConfig,
-        vertex_ai::ocr::transformation::VertexAiOcrConfig,
-    };
-    use litellm_llms_types::formats::ocr::OcrResponseFormat;
-    use serde_json::{Value, json};
-
-    use super::*;
-    use crate::{
-        document::prepare_document,
-        types::LiteLLMOcrRequest,
-        wire::{OcrWireRequest, decode_request},
-    };
-
-    /// Stands in for a host with no interceptors registered.
-    struct NoHooks;
-
-    impl CallHooks<Error> for NoHooks {
-        fn before_provider_request(
-            &self,
-            wire: WireRequest,
-        ) -> BoxFuture<'_, Result<WireRequest, Error>> {
-            Box::pin(async move { Ok(wire) })
-        }
-
-        fn response_received<'a>(&'a self, _body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
-            Box::pin(async { Ok(()) })
-        }
-    }
-
-    fn client() -> OcrClient {
-        OcrClient::for_test(
-            litellm_http::Client::plain_for_test(),
-            litellm_http::Client::no_redirect_for_test(),
-        )
-    }
-
-    fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
-        decode_request(OcrWireRequest {
-            model: model.into(),
-            document,
-            api_key: Some(litellm_auth::SecretValue::new("test-key")),
-            api_base: Some(base.into()),
-            custom_llm_provider: None,
-            extra_headers: None,
-            optional_params: options.as_object().unwrap().clone(),
-            input_sources: Default::default(),
-            timeout_seconds: Some(2.0),
-        })
-        .unwrap()
-    }
-
-    fn prepared(request: LiteLLMOcrRequest) -> PreparedOcrRequest {
-        prepare_request(
-            request.map_document(prepare_document).unwrap(),
-            true,
-            &client(),
-            std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
-        )
-    }
-
-    fn image(url: &str) -> Value {
-        json!({"type": "image_url", "image_url": url})
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
-        let request = request(
-            "cohere/parse",
-            "https://example.com",
-            image("https://example.com/original.png"),
-            json!({
-                "output_format": "markdown", "timeout": 30,
-                "extra_body": {
-                    "output_format": {"future": true},
-                    "document": {"type": "image_url", "image_url": "https://example.com/a.png",
-                        "provider_options": {"nested": [false, 0, null]}}
-                }
-            }),
-        );
-
-        let http = CohereParseConfig
-            .prepare_request(&prepared(request), &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        let body: Value = serde_json::from_slice(http.body()).unwrap();
-        assert_eq!(
-            body,
-            json!({
-                "model": "parse", "output_format": {"future": true},
-                "document": {"type": "image_url", "image_url": "https://example.com/a.png",
-                    "provider_options": {"nested": [false, 0, null]}}
-            })
-        );
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn explicit_null_options_use_defaults_before_http() {
-        let request = request(
-            "cohere/parse",
-            "https://example.com",
-            image("https://example.com/a.png"),
-            json!({"output_format": null, "req_format": null}),
-        );
-        assert_eq!(
-            request.response_format().unwrap(),
-            OcrResponseFormat::Litellm
-        );
-
-        let http = CohereParseConfig
-            .prepare_request(&prepared(request), &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        let body: Value = serde_json::from_slice(http.body()).unwrap();
-        assert_eq!(body["output_format"], "markdown");
-        assert!(body.get("req_format").is_none());
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
-        let options = json!({
-            "pages": [0, 2],
-            "include_image_base64": true,
-            "vertex_project": "project-1",
-            "vertex_location": "us-central1",
-            "unknown": "preserved"
-        });
-        let document =
-            json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
-        let direct = prepared(request(
-            "mistral/mistral-ocr-maas",
-            "https://mistral.test",
-            document.clone(),
-            options.clone(),
-        ));
-        let vertex = prepared(request(
-            "vertex_ai/mistral-ocr-maas",
-            "https://vertex.test",
-            document,
-            options,
-        ));
-
-        let direct_http = MistralOcrConfig
-            .prepare_request(&direct, &client(), &NoHooks)
-            .await
-            .unwrap();
-        let vertex_http = VertexAiOcrConfig
-            .prepare_request(&vertex, &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        assert_eq!(direct_http.url(), "https://mistral.test/v1/ocr");
-        assert_eq!(
-            vertex_http.url(),
-            "https://vertex.test/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
-        );
-        for http in [&direct_http, &vertex_http] {
-            assert_eq!(http.header("authorization").unwrap(), "Bearer test-key");
-            assert_eq!(http.header("content-type").unwrap(), "application/json");
-            assert_eq!(http.timeout(), Some(Duration::from_secs(2)));
-            let body: Value = serde_json::from_slice(http.body()).unwrap();
-            assert_eq!(
-                body,
-                json!({
-                    "model": "mistral-ocr-maas",
-                    "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
-                    "pages": [0, 2],
-                    "include_image_base64": true,
-                    "unknown": "preserved"
-                })
-            );
-        }
-        let payload = serde_json::to_vec(
-            &json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}),
-        )
-        .unwrap();
-        let direct_response = MistralOcrConfig
-            .transform_ocr_response(&direct.model, &payload, OcrResponseFormat::Litellm)
-            .unwrap()
-            .into_json();
-        let vertex_response = VertexAiOcrConfig
-            .transform_ocr_response(&vertex.model, &payload, OcrResponseFormat::Litellm)
-            .unwrap()
-            .into_json();
-        assert_eq!(direct_response, vertex_response);
-        assert_eq!(direct_response["model"], "mistral-ocr-maas");
-        assert_eq!(direct_response["object"], "ocr");
-        assert_eq!(direct_response["extra"], "preserved");
-    }
-
-    #[derive(serde::Deserialize)]
-    struct KnownParams {
-        pages: Option<Vec<i64>>,
-    }
-
-    #[rstest::rstest]
-    fn parsed_provider_params_separates_known_and_extra_params() {
-        let arguments: CallArguments = serde_json::from_value(json!({
-            "pages": [0, 2],
-            "future_ocr_option": true,
-            "extra_body": {"provider_option": "value"}
-        }))
-        .unwrap();
-        let known: KnownParams = parse_options(&arguments).unwrap();
-        assert_eq!(known.pages, Some(vec![0, 2]));
-        assert_eq!(arguments["future_ocr_option"], true);
-        assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
-        assert_eq!(
-            arguments
-                .iter()
-                .filter(|(name, _)| name.as_str() != "pages")
-                .count(),
-            2
-        );
-        assert_eq!(
-            compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
-            json!({
-                "pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
-            })
-        );
-    }
-}

litellm-rust/crates/inference-ocr/src/provider_config.rs

@@ -1,5 +1,6 @@
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use litellm_auth::{InputSource, SecretValue, Sourced};
-use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
+use litellm_core_utils::get_llm_provider_logic::{LlmProviders, get_custom_llm_provider};
+use litellm_inference::provider::{ResolvedProvider, resolve_llm_provider};
 use litellm_llms::{
     aws_textract::ocr::{
         analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation,
@@ -25,6 +26,7 @@ use litellm_llms::{
     },
 };
 use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat};
+use litellm_secrets::source::Secrets;
 
 macro_rules! with_config {
     ($kind:expr, $config:ident => $body:expr) => {
@@ -115,7 +117,18 @@ impl OcrConfigKind {
     }
 
     pub(crate) fn secret_names(self) -> Vec<&'static str> {
-        with_config!(self, config => config.secret_names())
+        let names = with_config!(self, config => config.secret_names());
+        let fallback_names: &[&str] = match self.provider() {
+            LlmProviders::Mistral => &["MISTRAL_AZURE_API_KEY", "MISTRAL_AZURE_API_BASE"],
+            LlmProviders::AzureAi => &["AZURE_AI_API_BASE"],
+            _ => &[],
+        };
+        let additional_names = fallback_names
+            .iter()
+            .copied()
+            .filter(|name| !names.contains(name))
+            .collect::<Vec<_>>();
+        names.into_iter().chain(additional_names).collect()
     }
 
     pub(crate) fn get_health_check_document(self) -> OcrDocument {
@@ -129,6 +142,43 @@ impl OcrConfigKind {
         with_config!(self, config => config.resolve_connection_params(inputs))
     }
 
+    pub(crate) fn resolve_credentials(
+        self,
+        credentials: OcrCredentialInputs,
+        secrets: &Secrets,
+    ) -> ResolvedOcrCredentials {
+        let (preferred_api_key_env, api_base_env) = match self.provider() {
+            LlmProviders::Mistral => (
+                Some("MISTRAL_AZURE_API_KEY"),
+                Some("MISTRAL_AZURE_API_BASE"),
+            ),
+            LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
+            _ => (None, None),
+        };
+        let secret = |name: &str| secrets.truthy(name);
+        let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
+            credentials.api_key.clone().or_else(|| {
+                preferred_api_key_env
+                    .into_iter()
+                    .chain(self.get_api_key_env_var())
+                    .find_map(secret)
+                    .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
+            })
+        });
+        let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
+            credentials.api_base.clone().or_else(|| {
+                api_base_env
+                    .and_then(secret)
+                    .map(|value| Sourced::new(value, InputSource::Environment))
+            })
+        });
+        self.resolve_connection_params(OcrCredentialInputs {
+            dynamic_api_key,
+            dynamic_api_base,
+            ..credentials
+        })
+    }
+
     pub(crate) async fn ocr(
         self,
         client: &OcrClient,
@@ -164,7 +214,7 @@ pub fn passthrough_response(
     endpoint: &str,
     body: &[u8],
 ) -> Result<Option<LiteLLMOcrResponse>, Error> {
-    let (model, config) = resolve_provider_config(model, Some("azure_ai"))?;
+    let (provider, config) = resolve_provider_config(model, Some("azure_ai"))?;
     let segments: Vec<&str> = endpoint
         .split('/')
         .filter(|segment| !segment.is_empty())
@@ -173,7 +223,7 @@ pub fn passthrough_response(
         OcrConfigKind::AzureAi => segments == AZURE_AI_OCR_PATH,
         OcrConfigKind::AzureCohere => segments == AZURE_COHERE_PARSE_PATH,
         OcrConfigKind::AzureDocumentIntelligence => {
-            segments == AzureDocumentIntelligenceOcrConfig::analyze_path(&model)?
+            segments == AzureDocumentIntelligenceOcrConfig::analyze_path(provider.model)?
         }
         other => {
             let provider: &'static str = other.provider().into();
@@ -183,24 +233,28 @@ pub fn passthrough_response(
     if !is_ocr_endpoint {
         return Ok(None);
     }
-    with_config!(config, config => config.transform_ocr_response(&model, body, OcrResponseFormat::Litellm))
+    with_config!(config, config => config.transform_ocr_response(provider.model, body, OcrResponseFormat::Litellm))
         .map(Some)
 }
 
-pub(crate) fn resolve_provider_config(
+pub(crate) fn resolve_provider_config<'a>(
-    model: &str,
+    model: &'a str,
-    custom_llm_provider: Option<&str>,
+    custom_llm_provider: Option<&'a str>,
-) -> Result<(String, OcrConfigKind), Error> {
+) -> Result<(ResolvedProvider<'a>, OcrConfigKind), Error> {
-    let provider =
+    let provider = match resolve_llm_provider(model, custom_llm_provider, "ocr") {
-        get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider {
+        Ok(provider) => provider,
-            model,
+        Err(_) if get_custom_llm_provider(model, custom_llm_provider).is_none() => {
-            custom_llm_provider: LlmProviders::Mistral.into(),
+            ResolvedProvider {
-        });
+                model,
-    let llm_provider = provider
+                provider: LlmProviders::Mistral,
-        .custom_llm_provider
+            }
-        .parse::<LlmProviders>()
+        }
-        .map_err(|_| Error::InvalidProvider(provider.custom_llm_provider.to_string()))?;
+        Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
-    let config = match llm_provider {
+            return Err(Error::InvalidProvider(provider));
+        }
+        Err(error) => return Err(Error::InvalidProvider(error.to_string())),
+    };
+    let config = match provider.provider {
         LlmProviders::AwsTextract => match TextractOperation::from_model(provider.model)? {
             TextractOperation::DetectDocumentText => OcrConfigKind::AwsTextract,
             TextractOperation::AnalyzeDocument => OcrConfigKind::AwsTextractAnalyze,
@@ -227,11 +281,11 @@ pub(crate) fn resolve_provider_config(
         LlmProviders::VertexAi => OcrConfigKind::VertexAi,
         _ => {
             return Err(Error::InvalidProvider(
-                provider.custom_llm_provider.to_string(),
+                <&'static str>::from(provider.provider).to_string(),
             ));
         }
     };
-    Ok((provider.model.to_string(), config))
+    Ok((provider, config))
 }
 
 fn is_document_intelligence_model(model: &str) -> bool {
@@ -243,7 +297,7 @@ fn is_document_intelligence_model(model: &str) -> bool {
 mod tests {
     use std::collections::HashSet;
 
-    use litellm_auth::{InputSource, Sourced};
+    use litellm_auth::Sourced;
     use litellm_llms::{
         base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document,
     };
@@ -484,8 +538,8 @@ mod tests {
         #[case] expected_config: OcrConfigKind,
     ) {
         let expected_model = qualified_model.split_once('/').unwrap().1;
-        let (model, config) = resolve_provider_config(qualified_model, None).unwrap();
+        let (provider, config) = resolve_provider_config(qualified_model, None).unwrap();
-        assert_eq!(model, expected_model);
+        assert_eq!(provider.model, expected_model);
         assert_eq!(config, expected_config);
     }
 
@@ -528,7 +582,7 @@ mod tests {
             expected_config
         );
         assert_eq!(
-            resolve_provider_config(model, None).unwrap().0,
+            resolve_provider_config(model, None).unwrap().0.model,
             model.split_once('/').unwrap().1
         );
     }

litellm-rust/crates/inference-ocr/src/types.rs

@@ -75,6 +75,8 @@ impl OcrConnectionInputs {
 
 pub struct LiteLLMOcrRequest<D = OcrDocumentInput> {
     pub model: String,
+    pub(crate) requested_model: String,
+    pub(crate) custom_llm_provider: Option<String>,
     pub document: D,
     pub credentials: OcrCredentialInputs,
     pub transport: OcrTransportConfig,
@@ -91,7 +93,8 @@ impl LiteLLMOcrRequest {
         custom_llm_provider: Option<&str>,
         optional_params: CallArguments,
     ) -> Result<Self, Error> {
-        let (model, config) = resolve_provider_config(&model, custom_llm_provider)?;
+        let (provider, config) = resolve_provider_config(&model, custom_llm_provider)?;
+        let resolved_model = provider.model.to_owned();
         let default_transport = OcrTransportConfig::default();
         let max_response_bytes = optional_params
             .get("max_response_bytes")
@@ -116,7 +119,9 @@ impl LiteLLMOcrRequest {
             .collect();
 
         Ok(Self {
-            model,
+            model: resolved_model,
+            requested_model: model,
+            custom_llm_provider: custom_llm_provider.map(str::to_owned),
             document: document.into(),
             credentials: OcrCredentialInputs::default(),
             transport,
@@ -135,6 +140,8 @@ impl<D> LiteLLMOcrRequest<D> {
     ) -> Result<LiteLLMOcrRequest<T>, E> {
         Ok(LiteLLMOcrRequest {
             model: self.model,
+            requested_model: self.requested_model,
+            custom_llm_provider: self.custom_llm_provider,
             document: map(self.document)?,
             credentials: self.credentials,
             transport: self.transport,
@@ -148,6 +155,8 @@ impl<D> LiteLLMOcrRequest<D> {
     pub fn with_document<T>(self, document: T) -> LiteLLMOcrRequest<T> {
         LiteLLMOcrRequest {
             model: self.model,
+            requested_model: self.requested_model,
+            custom_llm_provider: self.custom_llm_provider,
             document,
             credentials: self.credentials,
             transport: self.transport,

litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs

@@ -66,6 +66,19 @@ async fn interception_runs_once_and_observers_receive_ordered_success_events() {
     assert_eq!(received(&upstream).await.len(), 1);
 }
 
+#[rstest::rstest]
+#[case::mutated_model("model-b")]
+#[tokio::test]
+async fn provider_request_uses_the_request_model_after_it_is_mutated(#[case] model: &str) {
+    let upstream = upstream([pages_response()]).await;
+    let mut request = ocr_request("mistral/model-a", &upstream.uri(), json!({}));
+    request.model = model.into();
+
+    perform(request).await.unwrap();
+
+    assert_eq!(only_request(&upstream).await.json()["model"], json!(model));
+}
+
 #[rstest::rstest]
 #[tokio::test]
 async fn a_blocking_before_send_prevents_the_call_and_emits_one_failure() {

litellm-rust/crates/inference-responses/AGENTS.md

@@ -2,3 +2,4 @@
 - This crate owns Responses API call orchestration. Payloads belong in `litellm-llms-types`, transformations in `llms/src/base_llm/responses` and `llms/src/<provider>/responses`
 - The HTTP route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks, cached through `litellm_inference::caching::execute_streaming`
 - WebSocket sessions (`src/websocket.rs`) stay separate from the HTTP call driver because a connection can accept multiple requests while receiving events
+- `src/ids.rs` owns the LiteLLM-managed ids handed back to callers (`resp_`, `encitem_`, `litellm_enc:`, `cntr_`) and must stay fully compatible with `litellm/responses/utils.py`

litellm-rust/crates/inference-responses/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+base64.workspace = true
 bytes.workspace = true
 futures-util.workspace = true
 litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }

litellm-rust/crates/inference-responses/src/handler.rs

@@ -3,39 +3,61 @@ use litellm_host::observation::ObservationSender;
 use std::time::Duration;
 
 use futures_util::StreamExt;
-use litellm_host::interceptors::{Interceptors, RawResponse, WireRequest};
+use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
-use litellm_llms::base_llm::auth::{Authenticated, resolve_auth};
+use litellm_llms::base_llm::auth::{AuthScheme, Authenticated, resolve_auth};
+use litellm_secrets::source::Secrets;
 
 use super::{
-    Error,
+    Error, ResponsesRoute,
-    types::{ProviderResponsesRequest, ResponsesOutput, ResponsesStreamHead},
+    types::{ResponsesCall, ResponsesOutput, ResponsesProvider, ResponsesStreamHead},
 };
 
 pub(super) async fn execute(
-    http: &litellm_http::Client,
+    route: &ResponsesRoute,
-    auth: &litellm_auth::AuthServices,
+    ResponsesProvider { identity, config }: ResponsesProvider,
-    request: ProviderResponsesRequest,
+    call: ResponsesCall,
-    cache: Option<litellm_cache_response::ScopedCache>,
+    secrets: Secrets,
     cache_options: Option<litellm_cache_response::CachePolicy>,
     interceptors: &impl Interceptors<Error>,
     observers: Option<&ObservationSender>,
 ) -> Result<ResponsesOutput, Error> {
-    let authenticated = resolve_auth(auth, request.environment, &|_| None).await?;
+    let lookup = |name: &str| secrets.get(name);
-    let identity = litellm_host::interceptors::ProviderIdentity {
+    let environment = config.validate_environment(
-        model: request.context.model.clone(),
+        litellm_http::request::string_headers("responses", call.extra_headers)?,
-        provider: request.context.custom_llm_provider.clone(),
+        call.api_key.as_deref(),
+        &lookup,
+    )?;
+    let url = config.get_complete_url(call.api_base.as_deref(), &lookup);
+    let context = RequestContext {
+        model: identity.model.clone(),
+        custom_llm_provider: identity.provider.clone(),
+        optional_params: serde_json::Value::Object(call.optional_params.clone()),
+        secret_fields: Vec::new(),
+        api_key: match &environment.auth {
+            AuthScheme::Credential { secret, .. } => Some(secret.clone()),
+            _ => None,
+        },
     };
+    let body = config.transform_responses_api_request(
+        &identity.model,
+        call.input,
+        call.optional_params,
+    )?;
+    let authenticated = resolve_auth(&route.auth, environment, &|_| None).await?;
     let wire = interceptors
         .before_provider_request(
             WireRequest {
-                url: request.url,
+                url,
                 headers: authenticated.headers,
-                body: request.body,
+                body,
             },
-            request.context,
+            context,
         )
         .await?;
-    let cache = cache.filter(|_| authenticated.signer.is_none());
+    let cache = route
+        .cache
+        .clone()
+        .filter(|_| authenticated.signer.is_none());
     let cache_request = litellm_inference::caching::CacheRequest::from_wire(
         identity,
         cache.as_ref().map(|_| &wire),
@@ -59,9 +81,9 @@ pub(super) async fn execute(
                 },
                 wire.url,
                 &wire.body,
-                Some(request.timeout.unwrap_or(Duration::from_secs(600))),
+                Some(call.timeout.unwrap_or(Duration::from_secs(600))),
             )?;
-            let response = litellm_inference::outbound::send(outbound, http)
+            let response = litellm_inference::outbound::send(outbound, &route.http)
                 .await
                 .map_err(network)?;
             let status = response.status().as_u16();
@@ -103,8 +125,7 @@ pub(super) async fn execute(
                 .map_err(Error::post_call)?;
             let value = serde_json::from_str(&body)
                 .map_err(|error| Error::InvalidResponse(error.to_string().into()))?;
-            request
+            config
-                .config
                 .transform_response_api_response(value)
                 .map(ResponsesOutput::Complete)
                 .map_err(Error::from)

litellm-rust/crates/inference-responses/src/ids.rs

@@ -0,0 +1,499 @@
+//! LiteLLM-managed identifiers on Responses payloads, fully compatible with `litellm/responses/utils.py`
+//!
+//! LiteLLM rewrites some ids in a Responses API result before handing it back, so that a later
+//! request carrying those ids can be routed to the provider and deployment that produced them,
+//! and then restored to what the provider originally issued. The native and Python paths serve
+//! the same traffic, so an id minted by either one must decode on the other.
+//!
+//! Each managed id is a fixed prefix followed by standard, padded base64 of a `;`-separated
+//! plaintext:
+//!
+//! | Id | Applied to | Decoded plaintext |
+//! |---|---|---|
+//! | `resp_<b64>` | the response `id` | `litellm:custom_llm_provider:{provider};model_id:{model_id};response_id:{id}` |
+//! | `encitem_<b64>` | the `id` of an output item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` |
+//! | `litellm_enc:<b64>;{content}` | that item's `encrypted_content` | `<b64>` is `model_id:{model_id}`, and the original content follows the `;` unencoded |
+//! | `cntr_<b64>` | `container_id` on output items, `code_interpreter_call`, and message annotations | `litellm:custom_llm_provider:{provider};model_id:{model_id};container_id:{id}` |
+//!
+//! For example, OpenAI's `resp_0a1b2c3d4e5f` served without a deployment id becomes
+//! `resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm`,
+//! which decodes to `litellm:custom_llm_provider:openai;model_id:None;response_id:resp_0a1b2c3d4e5f`.
+//!
+//! How they fit together:
+//! - `resp_` lets a follow-up's `previous_response_id` find the provider and deployment again.
+//!   [`encode_response`] stamps it on every response, unless the id is already managed.
+//! - `encitem_` and `litellm_enc:` carry encrypted-content affinity: encrypted reasoning only
+//!   decrypts under the API key that produced it, so the router pins replays to that deployment.
+//!   They are stamped only when the router sets `encrypted_content_affinity_enabled` and a
+//!   deployment `model_id` is known (see [`IdScope`]). Items without an `id`, such as Codex's,
+//!   carry the deployment in the wrapped content alone.
+//! - `cntr_` lets container calls find the provider and deployment that own a container. A
+//!   container id that already names a provider is left as it is.
+//! - On the way in, [`restore_input`] turns `encitem_` ids and `litellm_enc:` content in the
+//!   request input back into the provider's own bytes before the request is sent. The routing
+//!   decision itself stays in Python's `EncryptedContentAffinityCheck`.
+//!
+//! Python's quirks are kept on purpose:
+//! - A missing provider or `model_id` is written as `None` in `resp_` ids (an f-string of
+//!   `None`), but as an empty string in `cntr_` ids. [`decode_container_id`] reads both as absent.
+//! - Decoding follows Python's non-strict `base64.b64decode`: characters outside the alphabet
+//!   are skipped, trailing bits are allowed, and padding stripped in transit is added back.
+//! - A `resp_` id counts as managed when it decodes to at least three `;`-separated parts.
+
+use base64::{
+    Engine, alphabet,
+    engine::{GeneralPurpose, general_purpose::PAD},
+};
+use litellm_llms_types::formats::responses::ResponsesApiResponse;
+use serde_json::{Map, Value};
+
+const RESPONSE_PREFIX: &str = "resp_";
+const ITEM_PREFIX: &str = "encitem_";
+const CONTAINER_PREFIX: &str = "cntr_";
+const WRAPPED_CONTENT_PREFIX: &str = "litellm_enc:";
+
+/// Python's non-strict `base64.b64decode`: characters outside the alphabet are skipped and
+/// non-zero trailing bits are accepted
+const LENIENT: GeneralPurpose = GeneralPurpose::new(
+    &alphabet::STANDARD,
+    PAD.with_decode_allow_trailing_bits(true),
+);
+
+/// Where a response came from, as encoded into the ids LiteLLM hands back
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct IdScope {
+    pub provider: Option<String>,
+    pub model_id: Option<String>,
+    pub encrypted_content_affinity: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DecodedContainerId {
+    pub provider: Option<String>,
+    pub model_id: Option<String>,
+    pub container_id: String,
+}
+
+/// Rewrites the response id, encrypted-content items and container ids the way the Python
+/// Responses path does before returning to the caller. An already managed id is left untouched
+pub fn encode_response(response: &mut ResponsesApiResponse, scope: &IdScope) {
+    if is_managed_response_id(&response.id) {
+        return;
+    }
+    response.id = response_id(scope, &response.id);
+    if let Some(model_id) = scope
+        .model_id
+        .as_deref()
+        .filter(|_| scope.encrypted_content_affinity)
+        .filter(|model_id| !model_id.is_empty())
+    {
+        response
+            .output
+            .iter_mut()
+            .filter_map(Value::as_object_mut)
+            .for_each(|item| encode_encrypted_item(item, model_id));
+    }
+    response
+        .output
+        .iter_mut()
+        .for_each(|item| encode_containers(item, scope));
+}
+
+/// Restores encrypted-content item ids and contents that LiteLLM encoded on an earlier response,
+/// so the provider receives what it originally issued
+pub fn restore_input(input: &mut Value) {
+    let Some(items) = input.as_array_mut() else {
+        return;
+    };
+    for item in items.iter_mut().filter_map(Value::as_object_mut) {
+        if let Some(Value::String(id)) = item.get_mut("id")
+            && let Some((_, original)) = decode_encrypted_item_id(id)
+        {
+            *id = original;
+        }
+        if let Some(Value::String(content)) = item.get_mut("encrypted_content")
+            && let Some(original) = unwrap_encrypted_content(content)
+        {
+            *content = original;
+        }
+    }
+}
+
+pub fn response_id(scope: &IdScope, id: &str) -> String {
+    let assembled = format!(
+        "litellm:custom_llm_provider:{};model_id:{};response_id:{id}",
+        python_str(scope.provider.as_deref()),
+        python_str(scope.model_id.as_deref()),
+    );
+    format!("{RESPONSE_PREFIX}{}", LENIENT.encode(assembled))
+}
+
+pub fn is_managed_response_id(id: &str) -> bool {
+    decode_text(&id.replace(RESPONSE_PREFIX, ""))
+        .is_some_and(|decoded| decoded.split(';').count() >= 3)
+}
+
+pub fn encrypted_item_id(model_id: &str, id: &str) -> String {
+    format!(
+        "{ITEM_PREFIX}{}",
+        LENIENT.encode(format!("litellm:model_id:{model_id};item_id:{id}"))
+    )
+}
+
+pub fn decode_encrypted_item_id(id: &str) -> Option<(String, String)> {
+    let decoded = decode_text(&padded(id.strip_prefix(ITEM_PREFIX)?))?;
+    let (model_id, item_id) = decoded.split_once(';')?;
+    Some((
+        model_id.replace("litellm:model_id:", ""),
+        item_id.replace("item_id:", ""),
+    ))
+}
+
+pub fn wrap_encrypted_content(content: &str, model_id: &str) -> String {
+    format!(
+        "{WRAPPED_CONTENT_PREFIX}{};{content}",
+        LENIENT.encode(format!("model_id:{model_id}"))
+    )
+}
+
+/// The original content of a wrapped `encrypted_content`, or `None` when it is not wrapped
+pub fn unwrap_encrypted_content(content: &str) -> Option<String> {
+    if !content.starts_with(WRAPPED_CONTENT_PREFIX) {
+        return None;
+    }
+    let (metadata, original) = content.split_once(';')?;
+    decode_text(&padded(&metadata.replace(WRAPPED_CONTENT_PREFIX, "")))?;
+    Some(original.to_owned())
+}
+
+pub fn container_id(scope: &IdScope, id: &str) -> String {
+    let assembled = format!(
+        "litellm:custom_llm_provider:{};model_id:{};container_id:{id}",
+        scope.provider.as_deref().unwrap_or_default(),
+        scope.model_id.as_deref().unwrap_or_default(),
+    );
+    format!("{CONTAINER_PREFIX}{}", LENIENT.encode(assembled))
+}
+
+pub fn decode_container_id(id: &str) -> DecodedContainerId {
+    let unmanaged = || DecodedContainerId {
+        provider: None,
+        model_id: None,
+        container_id: id.to_owned(),
+    };
+    if !id.starts_with(CONTAINER_PREFIX) {
+        return unmanaged();
+    }
+    let Some(decoded) = decode_text(&id.replace(CONTAINER_PREFIX, "")) else {
+        return unmanaged();
+    };
+    let parsed = decoded
+        .strip_prefix("litellm:custom_llm_provider:")
+        .and_then(|rest| rest.split_once(';'))
+        .and_then(|(provider, rest)| {
+            let (model_id, rest) = rest.strip_prefix("model_id:")?.split_once(';')?;
+            let container_id = rest.strip_prefix("container_id:")?;
+            (!container_id.is_empty() && !container_id.contains('\n')).then_some((
+                provider,
+                model_id,
+                container_id,
+            ))
+        });
+    let Some((provider, model_id, container_id)) = parsed else {
+        return unmanaged();
+    };
+    let present = |value: &str| (!matches!(value, "" | "None")).then(|| value.to_owned());
+    DecodedContainerId {
+        provider: present(provider),
+        model_id: present(model_id),
+        container_id: container_id.to_owned(),
+    }
+}
+
+fn encode_encrypted_item(item: &mut Map<String, Value>, model_id: &str) {
+    let Some(Value::String(content)) = item.get_mut("encrypted_content") else {
+        return;
+    };
+    if content.is_empty() {
+        return;
+    }
+    *content = wrap_encrypted_content(content, model_id);
+    if let Some(Value::String(id)) = item.get_mut("id")
+        && !id.is_empty()
+    {
+        *id = encrypted_item_id(model_id, id);
+    }
+}
+
+fn encode_containers(item: &mut Value, scope: &IdScope) {
+    let Some(item) = item.as_object_mut() else {
+        return;
+    };
+    encode_container_field(item, scope);
+    if let Some(nested) = item
+        .get_mut("code_interpreter_call")
+        .and_then(Value::as_object_mut)
+    {
+        encode_container_field(nested, scope);
+    }
+    if item.get("type").and_then(Value::as_str) != Some("message") {
+        return;
+    }
+    let Some(content) = item.get_mut("content").and_then(Value::as_array_mut) else {
+        return;
+    };
+    content
+        .iter_mut()
+        .filter_map(|part| part.get_mut("annotations").and_then(Value::as_array_mut))
+        .flatten()
+        .for_each(|annotation| encode_containers(annotation, scope));
+}
+
+fn encode_container_field(item: &mut Map<String, Value>, scope: &IdScope) {
+    if let Some(Value::String(id)) = item.get_mut("container_id")
+        && decode_container_id(id).provider.is_none()
+    {
+        *id = container_id(scope, id);
+    }
+}
+
+fn python_str(value: Option<&str>) -> &str {
+    value.unwrap_or("None")
+}
+
+fn padded(value: &str) -> String {
+    let missing = value.len() % 4;
+    let padding = if missing == 0 { 0 } else { 4 - missing };
+    format!("{value}{}", "=".repeat(padding))
+}
+
+fn decode_text(value: &str) -> Option<String> {
+    let alphabet: String = value
+        .chars()
+        .filter(|character| {
+            character.is_ascii_alphanumeric() || matches!(character, '+' | '/' | '=')
+        })
+        .collect();
+    String::from_utf8(LENIENT.decode(alphabet).ok()?).ok()
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    fn scope(provider: Option<&str>, model_id: Option<&str>, affinity: bool) -> IdScope {
+        IdScope {
+            provider: provider.map(str::to_owned),
+            model_id: model_id.map(str::to_owned),
+            encrypted_content_affinity: affinity,
+        }
+    }
+
+    fn response(id: &str, output: Value) -> ResponsesApiResponse {
+        serde_json::from_value(json!({"id": id, "model": "gpt-6.1-sol", "output": output})).unwrap()
+    }
+
+    #[rstest]
+    #[case::openai_without_model_id(
+        Some("openai"),
+        None,
+        "resp_0a1b2c3d4e5f",
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm"
+    )]
+    #[case::openrouter_with_model_id(
+        Some("openrouter"),
+        Some("deploy-1"),
+        "gen-123",
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5yb3V0ZXI7bW9kZWxfaWQ6ZGVwbG95LTE7cmVzcG9uc2VfaWQ6Z2VuLTEyMw=="
+    )]
+    fn response_ids_match_python(
+        #[case] provider: Option<&str>,
+        #[case] model_id: Option<&str>,
+        #[case] id: &str,
+        #[case] expected: &str,
+    ) {
+        assert_eq!(response_id(&scope(provider, model_id, false), id), expected);
+    }
+
+    #[rstest]
+    #[case::raw_short_id("resp_0a1b2c3d4e5f", false)]
+    #[case::raw_long_id("resp_68a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3", false)]
+    #[case::two_segments("resp_YTti", false)]
+    #[case::three_segments("resp_YTtiO2M=", true)]
+    #[case::managed(
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOng=",
+        true
+    )]
+    fn managed_response_ids_match_python(#[case] id: &str, #[case] expected: bool) {
+        assert_eq!(is_managed_response_id(id), expected);
+    }
+
+    #[test]
+    fn encrypted_items_match_python() {
+        assert_eq!(
+            encrypted_item_id("deploy-1", "rs_0a1b"),
+            "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI="
+        );
+        assert_eq!(
+            wrap_encrypted_content("gAAAA", "deploy-1"),
+            "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA"
+        );
+    }
+
+    #[rstest]
+    #[case::padded("encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", Some(("deploy-1", "rs_0a1b")))]
+    #[case::padding_stripped_in_transit("encitem_bGl0ZWxsbTptb2RlbF9pZDptO2l0ZW1faWQ6cnNfMQ", Some(("m", "rs_1")))]
+    #[case::not_base64("encitem_notbase64!", None)]
+    #[case::unmanaged("rs_0a1b", None)]
+    fn encrypted_item_ids_decode_like_python(
+        #[case] id: &str,
+        #[case] expected: Option<(&str, &str)>,
+    ) {
+        assert_eq!(
+            decode_encrypted_item_id(id),
+            expected.map(|(model_id, item_id)| (model_id.to_owned(), item_id.to_owned()))
+        );
+    }
+
+    #[rstest]
+    #[case::unpadded_metadata("litellm_enc:bW9kZWxfaWQ6bQ;gAAAA", Some("gAAAA"))]
+    #[case::no_separator("litellm_enc:nosemicolon", None)]
+    #[case::unwrapped("gAAAA", None)]
+    fn encrypted_content_unwraps_like_python(
+        #[case] content: &str,
+        #[case] expected: Option<&str>,
+    ) {
+        assert_eq!(unwrap_encrypted_content(content).as_deref(), expected);
+    }
+
+    #[rstest]
+    #[case::with_model_id(
+        Some("deploy-1"),
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpkZXBsb3ktMTtjb250YWluZXJfaWQ6Y250cl9hYmM="
+    )]
+    #[case::without_model_id(
+        None,
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfYWJj"
+    )]
+    fn container_ids_match_python(#[case] model_id: Option<&str>, #[case] expected: &str) {
+        assert_eq!(
+            container_id(&scope(Some("openai"), model_id, false), "cntr_abc"),
+            expected
+        );
+    }
+
+    #[rstest]
+    #[case::raw("cntr_abc", None, None, "cntr_abc")]
+    #[case::managed(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDptO2NvbnRhaW5lcl9pZDpjbnRyXzE=",
+        Some("openai"),
+        Some("m"),
+        "cntr_1"
+    )]
+    #[case::empty_segments(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOjttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfMQ==",
+        None,
+        None,
+        "cntr_1"
+    )]
+    #[case::python_none_provider(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOk5vbmU7bW9kZWxfaWQ6O2NvbnRhaW5lcl9pZDp4",
+        None,
+        None,
+        "x"
+    )]
+    fn container_ids_decode_like_python(
+        #[case] id: &str,
+        #[case] provider: Option<&str>,
+        #[case] model_id: Option<&str>,
+        #[case] container_id: &str,
+    ) {
+        assert_eq!(
+            decode_container_id(id),
+            DecodedContainerId {
+                provider: provider.map(str::to_owned),
+                model_id: model_id.map(str::to_owned),
+                container_id: container_id.to_owned(),
+            }
+        );
+    }
+
+    #[test]
+    fn encodes_response_items_under_affinity() {
+        let mut encoded = response(
+            "resp_1",
+            json!([
+                {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
+                {"type": "reasoning", "encrypted_content": "gBBBB", "summary": []},
+                {"type": "code_interpreter_call", "id": "ci_1", "container_id": "cntr_abc"},
+                {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
+                    {"type": "container_file_citation", "container_id": "cntr_abc"}
+                ]}]}
+            ]),
+        );
+        let scope = scope(Some("openai"), Some("deploy-1"), true);
+        encode_response(&mut encoded, &scope);
+        let container = container_id(&scope, "cntr_abc");
+        assert_eq!(encoded.id, response_id(&scope, "resp_1"));
+        assert_eq!(
+            Value::Array(encoded.output),
+            json!([
+                {
+                    "type": "reasoning",
+                    "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
+                    "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
+                    "summary": []
+                },
+                {"type": "reasoning", "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gBBBB", "summary": []},
+                {"type": "code_interpreter_call", "id": "ci_1", "container_id": container},
+                {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
+                    {"type": "container_file_citation", "container_id": container}
+                ]}]}
+            ])
+        );
+    }
+
+    #[rstest]
+    #[case::affinity_disabled(scope(Some("openai"), Some("deploy-1"), false))]
+    #[case::no_model_id(scope(Some("openai"), None, true))]
+    fn leaves_encrypted_items_without_affinity(#[case] scope: IdScope) {
+        let output = json!([{"type": "reasoning", "id": "rs_1", "encrypted_content": "gAAAA", "summary": []}]);
+        let mut encoded = response("resp_1", output.clone());
+        encode_response(&mut encoded, &scope);
+        assert_eq!(Value::Array(encoded.output), output);
+    }
+
+    #[test]
+    fn leaves_managed_responses_untouched() {
+        let scope = scope(Some("openai"), Some("deploy-1"), true);
+        let id = response_id(&scope, "resp_1");
+        let output = json!([{"type": "code_interpreter_call", "container_id": "cntr_abc"}]);
+        let mut encoded = response(&id, output.clone());
+        encode_response(&mut encoded, &scope);
+        assert_eq!((encoded.id, Value::Array(encoded.output)), (id, output));
+    }
+
+    #[test]
+    fn restores_encrypted_items_in_input() {
+        let mut input = json!([
+            {
+                "type": "reasoning",
+                "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
+                "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
+                "summary": []
+            },
+            {"role": "user", "content": "hi"}
+        ]);
+        restore_input(&mut input);
+        assert_eq!(
+            input,
+            json!([
+                {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
+                {"role": "user", "content": "hi"}
+            ])
+        );
+    }
+}

litellm-rust/crates/inference-responses/src/lib.rs

@@ -1,19 +1,36 @@
 use litellm_host::observation::ObservationSender;
 pub use litellm_inference::RouteError as Error;
 
+pub mod ids;
 pub mod route;
 pub mod types;
 pub mod websocket;
 
 mod handler;
-mod prepare;
 
 use std::sync::Arc;
 
 use litellm_auth::AuthServices;
+use litellm_core_utils::get_llm_provider_logic::{
+    CustomLlmProvider, LlmProviders, get_custom_llm_provider,
+};
 use litellm_host::interceptors::Interceptors;
+use litellm_host::interceptors::ProviderIdentity;
+use litellm_llms::{
+    base_llm::responses::transformation::BaseResponsesApiConfig,
+    openai::responses::transformation::OpenAiResponsesApiConfig,
+    openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
+};
 use litellm_secrets::source::SecretSource;
-use types::{ResponsesCall, ResponsesOutput};
+use types::{ResponsesCall, ResponsesOutput, ResponsesProvider};
+
+fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> {
+    match provider {
+        LlmProviders::Openai => Some(&OpenAiResponsesApiConfig),
+        LlmProviders::Openrouter => Some(&OPENROUTER_RESPONSES_CONFIG),
+        _ => None,
+    }
+}
 
 #[derive(Clone)]
 pub struct ResponsesRoute {
@@ -77,34 +94,68 @@ impl ResponsesRoute {
         observers: Option<&ObservationSender>,
     ) -> Result<ResponsesOutput, Error> {
         litellm_inference::diagnostic::call(async {
-            self.run_provider(call, cache_options, interceptors, observers)
+            let explicit = call
-                .await
+                .custom_llm_provider
+                .as_deref()
+                .filter(|provider| !provider.is_empty());
+            let (provider_name, resolved_model) =
+                match get_custom_llm_provider(&call.model, explicit) {
+                    Some(CustomLlmProvider {
+                        model,
+                        custom_llm_provider,
+                    }) => (custom_llm_provider, model),
+                    None => ("openai", call.model.as_str()),
+                };
+            if resolved_model.is_empty() {
+                return Err(Error::InvalidProvider(call.model));
+            }
+            let provider = provider_name
+                .parse::<LlmProviders>()
+                .map_err(|_| Error::Unsupported("native HTTP responses provider"))?;
+            let config = responses_provider(provider)
+                .ok_or(Error::Unsupported("native HTTP responses provider"))?;
+            let wire_model = match provider {
+                LlmProviders::Openai if resolved_model.contains('/') => {
+                    return Err(Error::InvalidProvider(call.model));
+                }
+                LlmProviders::Openai => resolved_model.into(),
+                LlmProviders::Openrouter if explicit.is_some() && !resolved_model.contains('/') => {
+                    format!("openrouter/{resolved_model}")
+                }
+                LlmProviders::Openrouter => resolved_model.into(),
+                _ => return Err(Error::Unsupported("native HTTP responses provider")),
+            };
+            let provider = ResponsesProvider {
+                identity: ProviderIdentity {
+                    model: wire_model,
+                    provider: provider_name.into(),
+                },
+                config,
+            };
+            litellm_inference::diagnostic::provider(
+                &provider.identity.model,
+                &provider.identity.provider,
+            );
+            let secrets = self
+                .secrets
+                .resolve(
+                    provider
+                        .config
+                        .secret_names(call.api_key.as_deref(), call.api_base.as_deref()),
+                )
+                .await?;
+            let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
+                Box::pin(handler::execute(
+                    self,
+                    provider,
+                    call,
+                    secrets,
+                    cache_options,
+                    interceptors,
+                    observers,
+                ));
+            execute.await
         })
         .await
     }
-
-    async fn run_provider(
-        &self,
-        call: ResponsesCall,
-        cache_options: Option<litellm_cache_response::CachePolicy>,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<ResponsesOutput, Error> {
-        let request = prepare::prepare(call, self.secrets.as_ref()).await?;
-        litellm_inference::diagnostic::provider(
-            &request.context.model,
-            &request.context.custom_llm_provider,
-        );
-        let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
-            Box::pin(handler::execute(
-                &self.http,
-                &self.auth,
-                request,
-                self.cache.clone(),
-                cache_options,
-                interceptors,
-                observers,
-            ));
-        execute.await
-    }
 }

litellm-rust/crates/inference-responses/src/prepare.rs

@@ -1,162 +0,0 @@
-use litellm_core_utils::get_llm_provider_logic::{
-    CustomLlmProvider, LlmProviders, get_custom_llm_provider,
-};
-use litellm_host::interceptors::{ProviderIdentity, RequestContext};
-use litellm_llms::{
-    base_llm::responses::transformation::BaseResponsesApiConfig,
-    openai::responses::transformation::OpenAiResponsesApiConfig,
-    openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
-};
-use litellm_secrets::source::SecretSource;
-use serde_json::Value;
-
-use super::{
-    Error,
-    types::{ProviderResponsesRequest, ResponsesCall},
-};
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) async fn prepare(
-    call: ResponsesCall,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderResponsesRequest, Error> {
-    let ResponsesProvider { identity, config } =
-        resolve_provider(&call.model, call.custom_llm_provider.as_deref())?;
-    let provider = identity.provider.as_str();
-    let model = identity.model.as_str();
-    let snapshot = secrets
-        .resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref()))
-        .await?;
-    let lookup = |name: &str| snapshot.get(name);
-    let environment = config.validate_environment(
-        litellm_http::request::string_headers("responses", call.extra_headers)?,
-        call.api_key.as_deref(),
-        &lookup,
-    )?;
-    let context = RequestContext {
-        model: model.into(),
-        custom_llm_provider: provider.into(),
-        optional_params: Value::Object(call.optional_params.clone()),
-        secret_fields: Vec::new(),
-        api_key: match &environment.auth {
-            litellm_llms::base_llm::auth::AuthScheme::Credential { secret, .. } => {
-                Some(secret.clone())
-            }
-            _ => None,
-        },
-    };
-    let body = config.transform_responses_api_request(model, call.input, call.optional_params)?;
-    Ok(ProviderResponsesRequest {
-        url: config.get_complete_url(call.api_base.as_deref(), &lookup),
-        config,
-        environment,
-        body,
-        context,
-        timeout: call.timeout,
-    })
-}
-
-pub(super) struct ResponsesProvider {
-    pub identity: ProviderIdentity,
-    pub config: &'static dyn BaseResponsesApiConfig,
-}
-
-/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter
-/// provider a model without a vendor segment keeps the `openrouter/` prefix as its wire id.
-pub(super) fn resolve_provider(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<ResponsesProvider, Error> {
-    let explicit = custom_llm_provider.filter(|provider| !provider.is_empty());
-    let (provider, resolved) = match get_custom_llm_provider(model, explicit) {
-        Some(CustomLlmProvider {
-            model,
-            custom_llm_provider,
-        }) => (custom_llm_provider, model),
-        None => ("openai", model),
-    };
-    if resolved.is_empty() {
-        return Err(Error::InvalidProvider(model.into()));
-    }
-    let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) =
-        match provider.parse::<LlmProviders>() {
-            Ok(LlmProviders::Openai) if resolved.contains('/') => {
-                return Err(Error::InvalidProvider(model.into()));
-            }
-            Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()),
-            Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => (
-                &OPENROUTER_RESPONSES_CONFIG,
-                format!("openrouter/{resolved}"),
-            ),
-            Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()),
-            _ => return Err(Error::Unsupported("native HTTP responses provider")),
-        };
-    Ok(ResponsesProvider {
-        identity: ProviderIdentity {
-            model: wire_model,
-            provider: provider.into(),
-        },
-        config,
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use rstest::rstest;
-
-    use super::*;
-
-    /// Pinned against Python's `get_llm_provider` on 2026-10-11.
-    #[rstest]
-    #[case::bare("gpt-5", None, "openai", "gpt-5")]
-    #[case::openai_prefix("openai/gpt-5", None, "openai", "gpt-5")]
-    #[case::openai_explicit("gpt-5", Some("openai"), "openai", "gpt-5")]
-    #[case::openrouter_prefix(
-        "openrouter/anthropic/claude",
-        None,
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_explicit(
-        "openrouter/anthropic/claude",
-        Some("openrouter"),
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_explicit_unprefixed(
-        "anthropic/claude",
-        Some("openrouter"),
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_native_id(
-        "openrouter/auto",
-        Some("openrouter"),
-        "openrouter",
-        "openrouter/auto"
-    )]
-    #[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")]
-    fn providers_and_wire_models_match_python(
-        #[case] model: &str,
-        #[case] custom_llm_provider: Option<&str>,
-        #[case] provider: &str,
-        #[case] wire_model: &str,
-    ) {
-        let resolved = resolve_provider(model, custom_llm_provider).unwrap();
-        assert_eq!(resolved.identity.provider, provider);
-        assert_eq!(resolved.identity.model, wire_model);
-    }
-
-    #[rstest]
-    #[case::other_prefix("other/model", None)]
-    #[case::other_explicit("model", Some("other"))]
-    #[case::openai_with_vendor_segment("openai/other/model", None)]
-    #[case::empty_model("openai/", None)]
-    #[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
-    fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) {
-        assert!(matches!(
-            resolve_provider(model, custom_llm_provider),
-            Err(Error::InvalidProvider(_) | Error::Unsupported(_))
-        ));
-    }
-}

litellm-rust/crates/inference-responses/src/types.rs

@@ -2,9 +2,8 @@ use std::time::Duration;
 
 use bytes::Bytes;
 use litellm_host::call::CallOutput;
-use litellm_llms::base_llm::{
+use litellm_host::interceptors::ProviderIdentity;
-    auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig,
+use litellm_llms::base_llm::responses::transformation::BaseResponsesApiConfig;
-};
 use litellm_llms_types::formats::responses::ResponsesApiResponse;
 use serde_json::{Map, Value};
 
@@ -21,17 +20,13 @@ pub struct ResponsesCall {
     pub timeout: Option<Duration>,
 }
 
+pub(crate) struct ResponsesProvider {
+    pub identity: ProviderIdentity,
+    pub config: &'static dyn BaseResponsesApiConfig,
+}
+
 pub struct ResponsesStreamHead {
     pub headers: Vec<(String, String)>,
 }
 
 pub type ResponsesOutput = CallOutput<ResponsesApiResponse, ResponsesStreamHead, Bytes, Error>;
-
-pub(super) struct ProviderResponsesRequest {
-    pub config: &'static dyn BaseResponsesApiConfig,
-    pub environment: ValidatedEnvironment,
-    pub url: String,
-    pub body: Value,
-    pub context: litellm_host::interceptors::RequestContext,
-    pub timeout: Option<Duration>,
-}

litellm-rust/crates/inference-responses/tests/responses.rs

@@ -7,6 +7,7 @@ use std::sync::Arc;
 use futures_util::TryStreamExt;
 use litellm_host::{call::HostedCompletion, lifecycle::CallEvent};
 use litellm_inference_responses::{
+    Error,
     route::Responses,
     types::{ResponsesCall, ResponsesOutput},
 };
@@ -32,6 +33,110 @@ fn call() -> ResponsesCall {
     }
 }
 
+#[rstest]
+#[case::bare_openai("gpt-5", None, "gpt-5")]
+#[case::openai_prefix("openai/gpt-5", None, "gpt-5")]
+#[case::empty_explicit_provider("openai/gpt-5", Some(""), "gpt-5")]
+#[case::explicit_openai("gpt-5", Some("openai"), "gpt-5")]
+#[case::openrouter_prefix("openrouter/anthropic/claude", None, "anthropic/claude")]
+#[case::explicit_openrouter_prefix(
+    "openrouter/anthropic/claude",
+    Some("openrouter"),
+    "anthropic/claude"
+)]
+#[case::explicit_openrouter_unprefixed("anthropic/claude", Some("openrouter"), "anthropic/claude")]
+#[case::explicit_openrouter_native_id("auto", Some("openrouter"), "openrouter/auto")]
+#[case::implicit_openrouter_native_id("openrouter/auto", None, "auto")]
+#[tokio::test]
+async fn provider_resolution_preserves_the_wire_model(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+    #[case] expected_wire_model: &str,
+) {
+    let response = json!({
+        "id": "response-1",
+        "model": expected_wire_model,
+        "output": [{"type": "message", "content": []}],
+        "usage": {"total_tokens": 7}
+    });
+    let upstream = upstream([json_response(response)]).await;
+    responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_key: Some("test-key".into()),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(
+        only_request(&upstream).await.json()["model"],
+        expected_wire_model
+    );
+}
+
+#[rstest]
+#[case::explicit_unknown_provider("test-model", Some("unknown-provider"))]
+#[case::unsupported_responses_provider("test-model", Some("anthropic"))]
+#[tokio::test]
+async fn unsupported_providers_fail_before_transport(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+) {
+    let upstream = upstream([]).await;
+    let result = responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await;
+
+    assert!(matches!(
+        result,
+        Err(Error::Unsupported("native HTTP responses provider"))
+    ));
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[case::openai_with_vendor_segment("openai/other/model", None)]
+#[case::empty_model("openai/", None)]
+#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
+#[tokio::test]
+async fn invalid_provider_models_fail_before_transport(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+) {
+    let upstream = upstream([]).await;
+    let result = responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await;
+
+    assert!(matches!(result, Err(Error::InvalidProvider(_))));
+    assert!(received(&upstream).await.is_empty());
+}
+
 #[rstest]
 #[case::direct(false)]
 #[case::hosted(true)]

litellm-rust/crates/inference-transcription/src/handler.rs

@@ -1,28 +1,57 @@
 use std::time::Duration;
 
-use litellm_http::{Client, request::truncate_error_body};
+use litellm_http::{
-use litellm_llms::base_llm::auth::resolve_auth;
+    Client,
+    request::{string_headers, truncate_error_body, with_default_headers},
+};
+use litellm_llms::base_llm::{
+    audio_transcription::transformation::BaseAudioTranscriptionConfig,
+    auth::{ValidatedEnvironment, resolve_auth},
+};
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
 use super::Error;
-use crate::{
+use crate::{constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::AudioTranscriptionRequest};
-    constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::ProviderAudioTranscriptionRequest,
+use litellm_inference::provider::ResolvedProvider;
-};
 
-pub async fn execute_audio_transcription_provider_call(
+pub(super) async fn execute(
     http: &Client,
     auth: &litellm_auth::AuthServices,
-    request: ProviderAudioTranscriptionRequest,
+    config: &'static dyn BaseAudioTranscriptionConfig,
+    provider: ResolvedProvider<'_>,
+    call: AudioTranscriptionRequest<'_>,
+    secrets: Secrets,
 ) -> Result<Value, Error> {
-    let env_lookup = |key: &str| request.secrets.get(key);
+    let env_lookup = |key: &str| secrets.get(key);
-    let authenticated = resolve_auth(auth, request.environment.clone(), &env_lookup).await?;
+    let forwarded = string_headers("audio transcription", call.extra_headers)?;
+    let validated = config.validate_environment(
+        forwarded,
+        provider.model,
+        &call.optional_params,
+        &env_lookup,
+    )?;
+    let environment = ValidatedEnvironment {
+        headers: with_default_headers(validated.headers, config.default_headers()),
+        auth: validated.auth,
+    };
+    let url = config.get_complete_url(
+        call.api_base,
+        provider.model,
+        &call.optional_params,
+        &env_lookup,
+    )?;
+    let filtered_params = config.map_transcription_params(&call.optional_params);
+    let body = config
+        .transform_audio_transcription_request(provider.model, call.audio, filtered_params)?
+        .body;
+    let authenticated = resolve_auth(auth, environment, &env_lookup).await?;
     let outbound = litellm_inference::outbound::outbound_request(
         authenticated,
-        request.url.clone(),
+        url,
-        &request.body,
+        &body,
         Some(
-            request
+            call.timeout
-                .timeout
                 .unwrap_or(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)),
         ),
     )?;
@@ -47,8 +76,7 @@ pub async fn execute_audio_transcription_provider_call(
             error,
         ))
     })?;
-    Ok(request
+    Ok(config
-        .config
+        .transform_audio_transcription_response(provider.model, response_json)?
-        .transform_audio_transcription_response(&request.model, response_json)?
         .into_json())
 }

litellm-rust/crates/inference-transcription/src/lib.rs

@@ -2,16 +2,29 @@ pub mod types;
 pub use litellm_inference::RouteError as Error;
 mod constants;
 mod handler;
-mod prepare;
+
-pub use handler::execute_audio_transcription_provider_call;
 use litellm_auth::AuthServices;
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use litellm_inference::provider::resolve_llm_provider;
+use litellm_llms::{
+    base_llm::audio_transcription::transformation::BaseAudioTranscriptionConfig,
+    bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
+};
 use litellm_secrets::source::SecretSource;
-pub use prepare::prepare_audio_transcription_provider_call;
 use serde_json::Value;
 use std::sync::Arc;
 
 use crate::types::AudioTranscriptionRequest;
 
+fn audio_transcription_provider(
+    provider: LlmProviders,
+) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
+    match provider {
+        LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
+        _ => None,
+    }
+}
+
 #[derive(Clone)]
 pub struct AudioTranscriptionRoute {
     http: litellm_http::Client,
@@ -32,6 +45,10 @@ impl AudioTranscriptionRoute {
         }
     }
 
+    pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
+        self.run(request).await
+    }
+
     #[tracing::instrument(name = "litellm.route", skip_all, fields(
         route = "audio_transcription",
         model = request.model,
@@ -40,13 +57,23 @@ impl AudioTranscriptionRoute {
         stream = false,
         outcome
     ))]
-    pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
+    async fn run(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
         litellm_inference::diagnostic::unary(async {
-            let request =
+            let provider = resolve_llm_provider(
-                prepare_audio_transcription_provider_call(request, self.secrets.as_ref()).await?;
+                request.model,
-            litellm_inference::diagnostic::provider(&request.model, &request.custom_llm_provider);
+                request.custom_llm_provider,
+                "audio transcription",
+            )?;
+            let config = audio_transcription_provider(provider.provider).ok_or_else(|| {
+                Error::InvalidProvider(<&str>::from(provider.provider).to_string())
+            })?;
+            litellm_inference::diagnostic::provider(
+                provider.model,
+                <&'static str>::from(provider.provider),
+            );
+            let secrets = self.secrets.resolve(&config.secret_names()).await?;
             let execute: futures_util::future::BoxFuture<'_, Result<Value, Error>> = Box::pin(
-                execute_audio_transcription_provider_call(&self.http, &self.auth, request),
+                handler::execute(&self.http, &self.auth, config, provider, request, secrets),
             );
             execute.await
         })

litellm-rust/crates/inference-transcription/src/prepare.rs

@@ -1,65 +0,0 @@
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_http::request::string_headers;
-use litellm_http::request::with_default_headers;
-use litellm_llms::{
-    base_llm::{
-        audio_transcription::transformation::BaseAudioTranscriptionConfig,
-        auth::ValidatedEnvironment,
-    },
-    bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
-};
-use litellm_secrets::source::SecretSource;
-
-use super::Error;
-use crate::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest};
-use litellm_inference::provider::resolve_llm_provider;
-
-fn provider_config(provider: LlmProviders) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
-    match provider {
-        LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
-        _ => None,
-    }
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub async fn prepare_audio_transcription_provider_call(
-    request: AudioTranscriptionRequest<'_>,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderAudioTranscriptionRequest, Error> {
-    let provider_info = resolve_llm_provider(
-        request.model,
-        request.custom_llm_provider,
-        "audio transcription",
-    )?;
-    let model = provider_info.model.to_string();
-    let config = provider_config(provider_info.provider)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?;
-    let snapshot = secrets.resolve(&config.secret_names()).await?;
-    let env_lookup = |key: &str| snapshot.get(key);
-    let forwarded = string_headers("audio transcription", request.extra_headers)?;
-    let validated =
-        config.validate_environment(forwarded, &model, &request.optional_params, &env_lookup)?;
-    let environment = ValidatedEnvironment {
-        headers: with_default_headers(validated.headers, config.default_headers()),
-        auth: validated.auth,
-    };
-    let url = config.get_complete_url(
-        request.api_base,
-        &model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    let filtered_params = config.map_transcription_params(&request.optional_params);
-    let transformed =
-        config.transform_audio_transcription_request(&model, request.audio, filtered_params)?;
-    Ok(ProviderAudioTranscriptionRequest {
-        model,
-        custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
-        config,
-        url,
-        body: transformed.body,
-        environment,
-        secrets: snapshot,
-        timeout: request.timeout,
-    })
-}

litellm-rust/crates/inference-transcription/src/types.rs

@@ -1,9 +1,5 @@
-use litellm_secrets::source::Secrets;
 use std::time::Duration;
 
-use litellm_llms::base_llm::{
-    audio_transcription::transformation::BaseAudioTranscriptionConfig, auth::ValidatedEnvironment,
-};
 use serde_json::{Map, Value};
 
 pub struct AudioTranscriptionRequest<'a> {
@@ -16,37 +12,3 @@ pub struct AudioTranscriptionRequest<'a> {
     pub optional_params: Map<String, Value>,
     pub timeout: Option<Duration>,
 }
-
-#[derive(Clone)]
-pub struct ProviderAudioTranscriptionRequest {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseAudioTranscriptionConfig,
-    pub url: String,
-    pub body: Value,
-    pub environment: ValidatedEnvironment,
-    pub secrets: Secrets,
-    pub timeout: Option<Duration>,
-}
-
-impl ProviderAudioTranscriptionRequest {
-    pub fn model(&self) -> &str {
-        &self.model
-    }
-
-    pub fn custom_llm_provider(&self) -> &str {
-        &self.custom_llm_provider
-    }
-
-    pub fn url(&self) -> &str {
-        &self.url
-    }
-
-    pub fn body(&self) -> &Value {
-        &self.body
-    }
-
-    pub fn with_body(self, body: Value) -> Self {
-        Self { body, ..self }
-    }
-}

litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs

@@ -195,9 +195,10 @@ impl AnthropicBeta {
 
     fn rejected_by(&self) -> &'static [BetaProvider] {
         match self {
-            Self::AdvancedToolUse20251120 | Self::ContextManagement20250627 => {
+            Self::AdvancedToolUse20251120 => {
-                &[BetaProvider::BedrockConverse]
+                &[BetaProvider::BedrockConverse, BetaProvider::Databricks]
             }
+            Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
             Self::AdvisorTool20260301 | Self::Compact20260904 => &[
                 BetaProvider::AzureAi,
                 BetaProvider::BedrockConverse,
@@ -231,12 +232,11 @@ impl AnthropicBeta {
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::VertexAi,
+                BetaProvider::Databricks,
             ],
             Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
             Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
-            Self::DangerousToolUse20260903 => {
+            Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
-                &[BetaProvider::BedrockConverse, BetaProvider::Databricks]
-            }
             Self::Effort20251124 => &[BetaProvider::VertexAi],
             Self::FastMode20260201 | Self::Oauth20250420 => &[
                 BetaProvider::AzureAi,
@@ -244,6 +244,7 @@ impl AnthropicBeta {
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::VertexAi,
+                BetaProvider::Databricks,
             ],
             Self::FineGrainedToolStreaming20250514 => {
                 &[BetaProvider::AzureAi, BetaProvider::VertexAi]

litellm-rust/crates/llms/src/anthropic/beta_headers.rs

@@ -76,7 +76,7 @@ mod tests {
     #[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))]
     #[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))]
     #[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))]
-    #[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))]
+    #[case::databricks(BetaProvider::Databricks, None)]
     #[case::bedrock_converse(BetaProvider::BedrockConverse, None)]
     fn filter_renames_per_host(
         #[case] provider: BetaProvider,

litellm-rust/crates/llms/src/anthropic/messages/transformation.rs

@@ -74,6 +74,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -109,7 +110,10 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
     }
 
     fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
-        update_headers_with_anthropic_beta(headers, request)
+        merge_beta_headers(
+            headers,
+            provider_feature_betas(request, BetaProvider::Anthropic),
+        )
     }
 }
 
@@ -136,12 +140,20 @@ pub(crate) fn transform_messages_request(
         messages: strip_encrypted_reasoning_blocks(messages),
         params: MessagesOptionalParams {
             context_management,
+            extra: without_null_safeguards(request.params.extra),
             ..request.params
         },
         ..request
     })
 }
 
+fn without_null_safeguards(extra: Map<String, Value>) -> Map<String, Value> {
+    extra
+        .into_iter()
+        .filter(|(key, value)| key != "safeguards" || !value.is_null())
+        .collect()
+}
+
 pub(crate) fn update_headers_with_anthropic_beta(
     headers: Headers,
     request: &MessagesRequest,
@@ -149,12 +161,34 @@ pub(crate) fn update_headers_with_anthropic_beta(
     merge_beta_headers(headers, feature_betas(request))
 }
 
-/// The betas a request needs on a host other than the first-party API: the Anthropic set
+pub(crate) fn update_headers_with_safeguards_beta(
-/// filtered and renamed through the host's `BetaProvider` policy.
+    headers: Headers,
+    request: &MessagesRequest,
+    provider: BetaProvider,
+) -> Headers {
+    merge_beta_headers(
+        headers,
+        feature_betas(request).union(on_provider(mapped_host_betas(&request.params), provider)),
+    )
+}
+
+/// The betas a request needs on a host with a column in the beta map, filtered and renamed
+/// through that column.
 pub(crate) fn provider_feature_betas(request: &MessagesRequest, provider: BetaProvider) -> BetaSet {
-    feature_betas(request)
+    on_provider(
-        .iter()
+        feature_betas(request).union(mapped_host_betas(&request.params)),
-        .filter_map(|beta| beta.on(provider))
+        provider,
+    )
+}
+
+fn on_provider(betas: BetaSet, provider: BetaProvider) -> BetaSet {
+    betas.iter().filter_map(|beta| beta.on(provider)).collect()
+}
+
+fn mapped_host_betas(params: &MessagesOptionalParams) -> BetaSet {
+    carries_safeguards(params)
+        .then_some(AnthropicBeta::DangerousToolUse20260903)
+        .into_iter()
         .collect()
 }
 
@@ -206,6 +240,13 @@ fn uses_structured_output(params: &MessagesOptionalParams) -> bool {
             .is_some_and(|config| config.format.is_some())
 }
 
+fn carries_safeguards(params: &MessagesOptionalParams) -> bool {
+    params
+        .extra
+        .get("safeguards")
+        .is_some_and(|value| !value.is_null())
+}
+
 fn messages_carry_output_config(messages: &[Message]) -> bool {
     messages
         .iter()
@@ -847,6 +888,7 @@ mod tests {
             headers(forwarded),
             api_key,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &env(vars),
         )
@@ -1025,10 +1067,44 @@ mod tests {
         json!({"messages": [{"role": "user", "content": "hi", "output_config": null}]}),
         &["per-turn-control-2026-07-01"]
     )]
+    #[case::safeguards(
+        json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
+        &[]
+    )]
     fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
         assert_eq!(feature_betas(&request(fields)), betas(expected));
     }
 
+    #[rstest]
+    #[case::anthropic(BetaProvider::Anthropic, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
+    #[case::vertex_ai(BetaProvider::VertexAi, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
+    #[case::empty_list(BetaProvider::Anthropic, json!([]), &["dangerous-tool-use-2026-09-03"])]
+    #[case::not_a_list(BetaProvider::Anthropic, json!(1), &["dangerous-tool-use-2026-09-03"])]
+    #[case::null(BetaProvider::Anthropic, Value::Null, &[])]
+    #[case::bedrock_converse_has_a_null_cell(BetaProvider::BedrockConverse, json!([{"type": "dangerous_tool_use"}]), &[])]
+    fn safeguards_add_the_dangerous_tool_use_beta_where_the_beta_map_has_a_cell(
+        #[case] provider: BetaProvider,
+        #[case] safeguards: Value,
+        #[case] expected: &[&str],
+    ) {
+        assert_eq!(
+            provider_feature_betas(&request(json!({"safeguards": safeguards})), provider),
+            betas(expected)
+        );
+    }
+
+    #[rstest]
+    fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
+        let client = headers(&[("anthropic-beta", "claude-code-20250219")]);
+        assert_eq!(
+            update_headers_with_anthropic_beta(
+                client.clone(),
+                &request(json!({"safeguards": [{"type": "dangerous_tool_use"}]}))
+            ),
+            client
+        );
+    }
+
     #[rstest]
     #[case::no_betas(&[("x-api-key", "k"), ("anthropic-version", "2023-06-01")], json!({}), &[("x-api-key", "k"), ("anthropic-version", "2023-06-01")])]
     #[case::blank_beta_header(&[("Anthropic-Beta", " , "), ("x-api-key", "k")], json!({}), &[("Anthropic-Beta", " , "), ("x-api-key", "k")])]
@@ -1067,6 +1143,11 @@ mod tests {
         json!({"messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}]}),
         &[("anthropic-beta", "claude-code-20250219,context-management-2025-06-27,effort-2025-11-24,interleaved-thinking-2025-05-14,per-turn-control-2026-07-01")],
     )]
+    #[case::safeguards_add_the_dangerous_tool_use_beta_to_the_client_betas(
+        &[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")],
+        json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14")],
+    )]
     fn request_headers_merge_the_feature_betas(
         #[case] input: &[(&str, &str)],
         #[case] fields: Value,
@@ -1127,6 +1208,7 @@ mod tests {
             Vec::new(),
             None,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &record,
         );

litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs

@@ -1,8 +1,8 @@
 use litellm_auth::SecretValue;
 use litellm_http::request::{has_bearer_auth, has_header};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
-    providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
+    providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
 };
 use litellm_router_types::LitellmParams;
 
@@ -12,7 +12,7 @@ use crate::{
         common_utils::API_KEY_PLACEMENT,
         messages::{
             handler::shape_anthropic_messages_request,
-            transformation::{transform_messages_request, update_headers_with_anthropic_beta},
+            transformation::{transform_messages_request, update_headers_with_safeguards_beta},
         },
     },
     azure_ai::common_utils::{
@@ -83,6 +83,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -104,7 +105,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
     }
 
     fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
-        update_headers_with_anthropic_beta(headers, request)
+        update_headers_with_safeguards_beta(headers, request, BetaProvider::AzureAi)
     }
 }
 
@@ -237,6 +238,7 @@ mod tests {
                     .collect(),
                 api_key,
                 "claude",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|_| None,
             )
@@ -544,6 +546,12 @@ mod tests {
         &[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
     )]
     #[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
+    #[case::safeguards_need_the_dangerous_tool_use_beta(
+        json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[("anthropic-beta", "claude-code-20250219")],
+        &[("x-api-key", "k"), ("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03")]
+    )]
+    #[case::null_safeguards_need_no_beta(json!({"safeguards": null}), &[], &[("x-api-key", "k")])]
     fn request_headers_carry_the_anthropic_feature_betas(
         #[case] fields: serde_json::Value,
         #[case] forwarded: &[(&str, &str)],
@@ -616,6 +624,7 @@ mod tests {
             Vec::new(),
             None,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &record,
         );

litellm-rust/crates/llms/src/base_llm/messages/transformation.rs

@@ -1,4 +1,6 @@
-use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_llms_types::formats::messages::{
+    MessagesOptionalParams, MessagesRequest, MessagesResponse,
+};
 use litellm_router_types::LitellmParams;
 use serde_json::Value;
 
@@ -55,6 +57,7 @@ pub trait BaseMessagesConfig: Sync {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error>;
@@ -110,6 +113,7 @@ mod tests {
             headers: Headers,
             _api_key: Option<&str>,
             _model: &str,
+            _optional_params: &MessagesOptionalParams,
             _litellm_params: &LitellmParams,
             _env_lookup: &dyn Fn(&str) -> Option<String>,
         ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs

@@ -5,8 +5,8 @@ use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
 use litellm_core_utils::settings::resolve_non_empty;
 use litellm_http::request::{header_value, with_header, without_headers};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
-    providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
+    providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
 };
 use litellm_router_types::LitellmParams;
 use serde_json::Value;
@@ -15,7 +15,7 @@ use crate::{
     Error, ErrorDetail,
     anthropic::messages::{
         handler::shape_anthropic_messages_request,
-        transformation::{transform_messages_request, update_headers_with_anthropic_beta},
+        transformation::{transform_messages_request, update_headers_with_safeguards_beta},
     },
     base_llm::{
         auth::{AuthScheme, Headers, ValidatedEnvironment},
@@ -60,6 +60,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        optional_params: &MessagesOptionalParams,
         params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -67,11 +68,8 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
             .filter(|value| !value.trim().is_empty())
             .or_else(|| {
                 WORKSPACE_FIELDS.iter().find_map(|field| {
-                    params
+                    workspace_value(optional_params.extra.get(*field))
-                        .extra
+                        .or_else(|| workspace_value(params.extra.get(*field)))
-                        .get(*field)?
-                        .as_str()
-                        .filter(|value| !value.trim().is_empty())
                 })
             });
         let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS)
@@ -174,7 +172,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
             Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()),
             None => headers,
         };
-        update_headers_with_anthropic_beta(headers, request)
+        update_headers_with_safeguards_beta(headers, request, BetaProvider::Anthropic)
     }
 
     fn wire_body(&self, body: Value) -> Value {
@@ -182,6 +180,10 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
     }
 }
 
+fn workspace_value(value: Option<&Value>) -> Option<&str> {
+    value?.as_str().filter(|value| !value.trim().is_empty())
+}
+
 fn platform_region(
     base: Option<&str>,
     params: &LitellmParams,
@@ -260,6 +262,7 @@ mod tests {
                     vec![],
                     Some("key"),
                     "claude-test",
+                    &MessagesOptionalParams::default(),
                     &LitellmParams::default(),
                     &|_| None
                 )
@@ -280,6 +283,7 @@ mod tests {
                 ],
                 Some("explicit"),
                 "claude-test",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|name| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into()),
             )
@@ -345,7 +349,14 @@ mod tests {
             .get_complete_url(None, "claude-test", &params, false, &env)
             .unwrap();
         let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
-            .validate_environment(vec![], None, "claude-test", &params, &env)
+            .validate_environment(
+                vec![],
+                None,
+                "claude-test",
+                &MessagesOptionalParams::default(),
+                &params,
+                &env,
+            )
             .unwrap();
         assert_eq!(
             url,
@@ -356,6 +367,21 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case::safeguards(json!([{"type": "dangerous_tool_use"}]), Some("dangerous-tool-use-2026-09-03"))]
+    #[case::null_safeguards(Value::Null, None)]
+    fn safeguards_need_the_dangerous_tool_use_beta(
+        #[case] safeguards: Value,
+        #[case] expected: Option<&str>,
+    ) {
+        let request = serde_json::from_value(json!({"model": "claude-test", "max_tokens": 16, "messages": [], "safeguards": safeguards})).unwrap();
+        let headers = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.request_headers(vec![], &request);
+        assert_eq!(
+            header_value(&headers, "anthropic-beta").map(|value| value.to_string()),
+            expected.map(str::to_string)
+        );
+    }
+
     #[rstest]
     #[case::default(json!({}), false)]
     #[case::override_empty(json!({"claude_platform_unsupported_params": []}), true)]

litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs

@@ -16,7 +16,7 @@ use litellm_auth_aws::{
     resolve_bedrock_region,
 };
 use litellm_llms_types::formats::messages::{
-    MessagesRequest,
+    MessagesOptionalParams, MessagesRequest,
     streaming::{MessagesStreamEvent, MessagesStreamUsage},
 };
 use litellm_llms_types::providers::bedrock::{
@@ -146,6 +146,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        _optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -618,6 +619,7 @@ mod tests {
                 Vec::new(),
                 None,
                 "anthropic.claude-3",
+                &MessagesOptionalParams::default(),
                 &litellm_params,
                 &|_| None,
             )
@@ -658,6 +660,7 @@ mod tests {
                 vec![("authorization".into(), "Bearer forwarded".into())],
                 api_key,
                 "anthropic.claude-3",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &env,
             )

litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs

@@ -6,7 +6,7 @@ use litellm_auth_aws::bedrock_model_id_and_region;
 use litellm_core_utils::settings::resolve_non_empty;
 use litellm_http::request::{with_header, without_headers};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
     providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
 };
 use litellm_router_types::LitellmParams;
@@ -59,6 +59,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        _optional_params: &MessagesOptionalParams,
         params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -317,7 +318,14 @@ mod tests {
             "https://private.test/anthropic/v1/messages"
         );
         let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
-            .validate_environment(vec![], None, model, &params, &|_| None)
+            .validate_environment(
+                vec![],
+                None,
+                model,
+                &MessagesOptionalParams::default(),
+                &params,
+                &|_| None,
+            )
             .unwrap();
         assert!(
             matches!(environment.auth, AuthScheme::AwsSigV4 { region: actual, service: "bedrock", .. } if actual == region)
@@ -347,7 +355,14 @@ mod tests {
             .get_complete_url(None, "claude-test", &params, false, &|_| None)
             .unwrap();
         let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
-            .validate_environment(vec![], None, "claude-test", &params, &|_| None)
+            .validate_environment(
+                vec![],
+                None,
+                "claude-test",
+                &MessagesOptionalParams::default(),
+                &params,
+                &|_| None,
+            )
             .unwrap();
         assert_eq!(
             url,
@@ -401,6 +416,7 @@ mod tests {
                 ],
                 key,
                 "claude-test",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|name| match name {
                     "BEDROCK_MANTLE_API_KEY" => Some("mantle".into()),

litellm-rust/crates/llms/src/deepseek/messages/transformation.rs

@@ -104,6 +104,7 @@ impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -298,6 +299,7 @@ mod tests {
                 .collect(),
             api_key,
             "deepseek-v4-pro",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             env,
         )
@@ -451,4 +453,27 @@ mod tests {
             json!({"type": "enabled", "budget_tokens": 1024})
         );
     }
+
+    #[rstest]
+    #[case::without_client_betas(&[])]
+    #[case::with_client_betas(&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")])]
+    fn safeguards_and_client_betas_reach_deepseek_as_sent(#[case] client: &[(&str, &str)]) {
+        let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
+        let body = json!({
+            "model": "deepseek-v4-pro",
+            "max_tokens": 100,
+            "messages": [{"role": "user", "content": "hi"}],
+            "safeguards": safeguards
+        });
+        let headers: Headers = client
+            .iter()
+            .map(|(name, value)| (name.to_string(), value.to_string()))
+            .collect();
+        assert_eq!(
+            DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG
+                .request_headers(headers.clone(), &request_from(body.clone())),
+            headers
+        );
+        assert_eq!(transformed(body)["safeguards"], safeguards);
+    }
 }

litellm-rust/crates/llms/src/edenai/messages/transformation.rs

@@ -36,6 +36,7 @@ impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/minimax/messages/transformation.rs

@@ -42,6 +42,7 @@ impl BaseMessagesConfig for MinimaxMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -183,7 +184,14 @@ mod tests {
                 .flatten()
         };
         let environment = MINIMAX_MESSAGES_CONFIG
-            .validate_environment(vec![], key, "test-model", &Default::default(), &env)
+            .validate_environment(
+                vec![],
+                key,
+                "test-model",
+                &MessagesOptionalParams::default(),
+                &Default::default(),
+                &env,
+            )
             .unwrap();
         assert!(matches!(
             environment.auth,

litellm-rust/crates/llms/src/openrouter/messages/transformation.rs

@@ -34,6 +34,7 @@ impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/tencent/messages/transformation.rs

@@ -43,6 +43,7 @@ impl BaseMessagesConfig for TencentAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -229,7 +230,14 @@ mod tests {
                 .flatten()
         };
         let environment = TENCENT_MESSAGES_CONFIG
-            .validate_environment(vec![], key, "test-model", &Default::default(), &env)
+            .validate_environment(
+                vec![],
+                key,
+                "test-model",
+                &MessagesOptionalParams::default(),
+                &Default::default(),
+                &env,
+            )
             .unwrap();
         assert!(matches!(
             environment.auth,

litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs

@@ -114,6 +114,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
         headers: Headers,
         _api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         _env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -213,23 +214,14 @@ fn has_web_search_tool(tools: Option<&[Recognized<MessagesTool>]>) -> bool {
 }
 
 /// Python's Vertex `validate_anthropic_messages_environment` beta set: the shared feature
-/// betas under Vertex's policy, plus the web search and safeguards betas Vertex needs spelled
+/// betas under Vertex's policy, plus the web search beta Vertex needs spelled out.
-/// out.
 fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
     let params = &request.params;
     provider_feature_betas(request, BetaProvider::VertexAi).union(
-        [
+        has_web_search_tool(params.tools.as_deref())
-            has_web_search_tool(params.tools.as_deref())
+            .then_some(AnthropicBeta::WebSearch20250305)
-                .then_some(AnthropicBeta::WebSearch20250305),
+            .into_iter()
-            params
+            .collect(),
-                .extra
-                .get("safeguards")
-                .is_some_and(|value| !value.is_null())
-                .then_some(AnthropicBeta::DangerousToolUse20260903),
-        ]
-        .into_iter()
-        .flatten()
-        .collect(),
     )
 }
 
@@ -448,6 +440,7 @@ mod tests {
                     .collect(),
                 Some("ignored-api-key"),
                 "claude-sonnet-4-5",
+                &MessagesOptionalParams::default(),
                 &params(Some("proj"), None),
                 &no_env,
             )

litellm-rust/crates/model-catalog/AGENTS.md

@@ -2,5 +2,4 @@
 
 For `model_prices_and_context_window.json` validation, we should eventually:
 
-- Remove any schema file like `model_prices_and_context_window.schema.json`
 - Stop skipping this crate's tests

litellm-rust/crates/python-bridge/src/routes/responses/host.rs

@@ -1,22 +1,41 @@
 use std::convert::Infallible;
 
 use crate::routes::inference::InferenceHost;
-use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned};
+use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
-use litellm_inference_responses::{Error, route::Responses, types::ResponsesCall};
+use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned, from_py};
+use litellm_inference_responses::{
+    Error,
+    ids::{self, IdScope},
+    route::Responses,
+    types::ResponsesCall,
+};
 use pyo3::{
     gc::{PyTraverseError, PyVisit},
     prelude::*,
     types::PyDict,
 };
 
-pub(super) struct ResponsesPythonHost(pub InferenceHost);
+pub(super) struct ResponsesPythonHost {
+    host: InferenceHost,
+    ids: IdScope,
+}
+
+impl ResponsesPythonHost {
+    pub fn new(host: InferenceHost) -> Self {
+        Self {
+            host,
+            ids: IdScope::default(),
+        }
+    }
+}
 
 pub(super) fn project(
     host: &mut InferenceHost,
     py: Python<'_>,
     arguments: &Bound<'_, PyDict>,
 ) -> PyResult<ResponsesCall> {
-    let call = host.project(py, arguments, "input")?;
+    let mut call = host.project(py, arguments, "input")?;
+    ids::restore_input(&mut call.input);
     let optional_params = call
         .params
         .into_iter()
@@ -50,6 +69,39 @@ pub(super) fn project(
     })
 }
 
+/// The provider and deployment Python encodes into the ids it returns: the resolved provider and
+/// `litellm_metadata.model_info.id`
+fn id_scope(
+    host: &InferenceHost,
+    py: Python<'_>,
+    arguments: &Bound<'_, PyDict>,
+    call: &ResponsesCall,
+) -> PyResult<IdScope> {
+    let metadata = host.argument(py, arguments, "litellm_metadata")?;
+    let encrypted_content_affinity = match &metadata {
+        Some(metadata) => metadata
+            .call_method1("get", ("encrypted_content_affinity_enabled",))?
+            .is_truthy()?,
+        None => false,
+    };
+    let metadata = metadata
+        .map(|value| from_py::<serde_json::Value>(&value))
+        .transpose()?
+        .unwrap_or_default();
+    let model_id = match metadata.pointer("/model_info/id") {
+        None | Some(serde_json::Value::Null) => None,
+        Some(serde_json::Value::String(id)) => Some(id.clone()),
+        Some(other) => Some(other.to_string()),
+    };
+    Ok(IdScope {
+        provider: get_custom_llm_provider(&call.model, call.custom_llm_provider.as_deref())
+            .map(|resolved| resolved.custom_llm_provider.to_owned())
+            .or_else(|| call.custom_llm_provider.clone()),
+        model_id,
+        encrypted_content_affinity,
+    })
+}
+
 impl PythonBinding for ResponsesPythonHost {
     type Protocol = Responses;
     type Failure = PyErr;
@@ -59,7 +111,8 @@ impl PythonBinding for ResponsesPythonHost {
         py: Python<'_>,
         arguments: &Bound<'_, PyDict>,
     ) -> Result<ResponsesCall, InvokeError<Error>> {
-        let call = project(&mut self.0, py, arguments).map_err(InvokeError::Python)?;
+        let call = project(&mut self.host, py, arguments).map_err(InvokeError::Python)?;
+        self.ids = id_scope(&self.host, py, arguments, &call).map_err(InvokeError::Python)?;
         if call
             .optional_params
             .get("stream")
@@ -77,7 +130,9 @@ impl PythonBinding for ResponsesPythonHost {
         py: Python<'_>,
         response: <Responses as litellm_host::protocol::Protocol>::Response,
     ) -> PyResult<Py<PyAny>> {
-        self.0.response(py, &response)
+        let mut response = response;
+        ids::encode_response(&mut response, &self.ids);
+        self.host.response(py, &response)
     }
 
     fn encode_stream_head(
@@ -103,7 +158,7 @@ impl PythonBinding for ResponsesPythonHost {
     }
 
     fn map_error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
-        self.0.error(py, error)
+        self.host.error(py, error)
     }
     fn host_error(error: &PyErr) -> Error {
         Error::InvalidRequest(error.to_string().into())
@@ -122,9 +177,9 @@ impl PythonHostCalls<Responses> for ResponsesPythonHost {
 
 impl PythonOwned for ResponsesPythonHost {
     fn close(&mut self, _: Python<'_>) {
-        self.0.close();
+        self.host.close();
     }
     fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
-        self.0.traverse(visit)
+        self.host.traverse(visit)
     }
 }

litellm-rust/crates/python-bridge/src/routes/responses/mod.rs

@@ -62,7 +62,7 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py
         ));
     }
     let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE);
-    run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost(host))
+    run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost::new(host))
 }
 
 #[pyfunction]

litellm/__init__.py

@@ -1,33 +1,13 @@
-### Hide pydantic namespace conflict warnings globally ###
 from __future__ import annotations
 
-import warnings
+from litellm._process_defaults import apply_process_defaults as _apply_process_defaults
+
+_apply_process_defaults()
 
-warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
-# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
-# This warning can accumulate during streaming and cause memory leaks
-warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
-# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
-# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
-warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
 ### INIT VARIABLES #########################
 import threading
 import os
-
+import datetime
-# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
-import dotenv as _dotenv
-
-
-def _dev_env_hot_reload_enabled() -> bool:
-    """The proxy exports this flag when started with ``--reload``. A reloaded
-    worker is a fresh process that inherits the reloader's environment, so an
-    edited ``.env`` value stays masked by the stale inherited one unless we
-    let the file win; overriding makes the edit take effect on reload."""
-    return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
-
-
-if os.getenv("LITELLM_MODE", "DEV") == "DEV":
-    _dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())
 
 from collections.abc import Mapping, Sequence
 from typing import (
@@ -83,7 +63,6 @@ from litellm.constants import (
     modelscope_models,
     empower_models,
     together_ai_models,
-    baseten_models,
     WANDB_MODELS,
     REPEATED_STREAMING_CHUNK_LIMIT,
     request_timeout,
@@ -375,6 +354,7 @@ llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all"
 guardrail_name_config_map: Dict[str, GuardrailItem] = {}
 include_cost_in_streaming_usage: bool = False
 reasoning_auto_summary: bool = False
+reasoning_effort_rounding: Literal["up", "down"] = "up"
 ### PROMPTS ####
 from litellm.types.prompts.init_prompts import PromptSpec
 
@@ -645,6 +625,7 @@ bedrock_models: Set = set()
 bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS)
 fal_ai_models: Set = set()
 fireworks_ai_models: Set = set()
+baseten_models: Set[str] = set()
 fireworks_ai_embedding_models: Set = set()
 deepinfra_models: Set = set()
 perplexity_models: Set = set()
@@ -697,6 +678,7 @@ qwen_ai_platform_models: Set = set()
 moonshot_models: Set = set()
 publicai_models: Set = set()
 darkbloom_models: Set = set()
+openinfer_models: Final[Set[str]] = set()
 v0_models: Set = set()
 morph_models: Set = set()
 lambda_ai_models: Set = set()
@@ -762,8 +744,18 @@ def is_openai_finetune_model(key: str) -> bool:
     return key.startswith("ft:") and not key.count(":") > 1
 
 
+def _is_past_deprecation_date(value: Mapping[str, object]) -> bool:
+    deprecation_date: Final = value.get("deprecation_date")
+    if not isinstance(deprecation_date, str):
+        return False
+    try:
+        return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today()
+    except ValueError:
+        return False
+
+
 def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
-    for key, value in model_cost_map.items():
+    for key, value in tuple(model_cost_map.items()):
         if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
             open_ai_chat_completion_models.add(key)
         elif value.get("litellm_provider") == "text-completion-openai":
@@ -862,6 +854,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
             if "-to-" not in key and "fireworks-ai-default" not in key:
                 fireworks_ai_models.add(key)
+        elif value.get("litellm_provider") == "baseten":
+            if _is_past_deprecation_date(value):
+                baseten_models.discard(key)
+            else:
+                baseten_models.add(key)
         elif value.get("litellm_provider") == "fireworks_ai-embedding-models":
             # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
             if "-to-" not in key:
@@ -966,6 +963,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             publicai_models.add(key)
         elif value.get("litellm_provider") == "darkbloom":
             darkbloom_models.add(key)
+        elif value.get("litellm_provider") == "openinfer":
+            openinfer_models.add(key)
         elif value.get("litellm_provider") == "v0":
             v0_models.add(key)
         elif value.get("litellm_provider") == "morph":
@@ -1134,6 +1133,7 @@ model_list = list(
     | moonshot_models
     | publicai_models
     | darkbloom_models
+    | openinfer_models
     | v0_models
     | morph_models
     | lambda_ai_models
@@ -1177,7 +1177,9 @@ def _build_models_by_provider() -> dict:
         "edenai": edenai_models,
         "datarobot": datarobot_models,
         "vertex_ai": vertex_chat_models
+        | vertex_code_chat_models
         | vertex_text_models
+        | vertex_code_text_models
         | vertex_anthropic_models
         | vertex_vision_models
         | vertex_language_models
@@ -1185,7 +1187,7 @@ def _build_models_by_provider() -> dict:
         | vertex_minimax_models
         | vertex_moonshot_models
         | vertex_zai_models,
-        "ai21": ai21_models,
+        "ai21": ai21_models | ai21_chat_models,
         "bedrock": bedrock_models | bedrock_converse_models,
         "petals": petals_models,
         "ollama": ollama_models,
@@ -1248,6 +1250,7 @@ def _build_models_by_provider() -> dict:
         "moonshot": moonshot_models,
         "publicai": publicai_models,
         "darkbloom": darkbloom_models,
+        "openinfer": openinfer_models,
         "v0": v0_models,
         "morph": morph_models,
         "lambda_ai": lambda_ai_models,
@@ -1692,6 +1695,9 @@ if TYPE_CHECKING:
     from .llms.perplexity.decisions.transformation import (
         PerplexityDecisionsConfig as PerplexityDecisionsConfig,
     )
+    from .llms.nace.decisions.transformation import (
+        NaceDecisionsConfig as NaceDecisionsConfig,
+    )
     from .llms.typesafe.decisions.transformation import (
         TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
     )

litellm/_lazy_imports_registry.py

@@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = (
     "HostedVLLMRerankConfig",
     "PerplexityDecisionsConfig",
     "TypeSafeDecisionsConfig",
+    "NaceDecisionsConfig",
     "OpenRouterDecisionsConfig",
     "FireworksAIDecisionsConfig",
     "CloudflareDecisionsConfig",
@@ -729,6 +730,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "HostedVLLMRerankConfig",
     ),
     "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
+    "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
     "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
     "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
     "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),

litellm/_logging.py

@@ -1060,6 +1060,8 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
         lg.handlers.clear()  # remove any existing handlers
         lg.addHandler(handler)  # add JSON formatter handler
         lg.propagate = False  # prevent bubbling to parent/root
+    if (configured_level := os.getenv("LITELLM_LOG")) is not None:
+        logging.getLogger().setLevel(resolve_log_level(configured_level))
 
 
 def _get_uvicorn_json_log_config():

litellm/_process_defaults.py

@@ -0,0 +1,24 @@
+import os
+import warnings
+
+import dotenv
+
+
+def _dev_env_hot_reload_enabled() -> bool:
+    """The proxy exports this flag when started with ``--reload``. A reloaded
+    worker is a fresh process that inherits the reloader's environment, so an
+    edited ``.env`` value stays masked by the stale inherited one unless we
+    let the file win; overriding makes the edit take effect on reload."""
+    return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
+
+
+def apply_process_defaults() -> None:
+    warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
+    # Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
+    # This warning can accumulate during streaming and cause memory leaks
+    warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
+    # ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
+    # cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
+    warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
+    if os.getenv("LITELLM_MODE", "DEV") == "DEV":
+        dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())

litellm/anthropic_beta_headers_config.json

@@ -220,36 +220,36 @@
   },
   "databricks": {
     "advisor-tool-2026-03-01": null,
-    "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
+    "advanced-tool-use-2025-11-20": null,
     "bash_20241022": null,
     "bash_20250124": null,
-    "code-execution-2025-08-25": "code-execution-2025-08-25",
+    "code-execution-2025-08-25": null,
     "compact-2026-01-12": "compact-2026-01-12",
     "computer-use-2025-01-24": "computer-use-2025-01-24",
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
-    "dangerous-tool-use-2026-09-03": null,
+    "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
-    "fast-mode-2026-02-01": "fast-mode-2026-02-01",
+    "fast-mode-2026-02-01": null,
-    "files-api-2025-04-14": "files-api-2025-04-14",
+    "files-api-2025-04-14": null,
     "structured-output-2024-03-01": null,
     "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
-    "mcp-client-2025-11-20": "mcp-client-2025-11-20",
+    "mcp-client-2025-11-20": null,
-    "mcp-client-2025-04-04": "mcp-client-2025-04-04",
+    "mcp-client-2025-04-04": null,
     "mcp-servers-2025-12-04": null,
-    "oauth-2025-04-20": "oauth-2025-04-20",
+    "oauth-2025-04-20": null,
     "output-128k-2025-02-19": "output-128k-2025-02-19",
     "per-turn-control-2026-07-01": null,
-    "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
+    "prompt-caching-scope-2026-01-05": null,
-    "skills-2025-10-02": "skills-2025-10-02",
+    "skills-2025-10-02": null,
     "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
     "text_editor_20241022": null,
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
-    "web-fetch-2025-09-10": "web-fetch-2025-09-10",
+    "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": "web-search-2025-03-05"
   }
 }

litellm/anthropic_beta_headers_manager.py

@@ -22,17 +22,28 @@ Environment Variables:
 - LITELLM_ANTHROPIC_BETA_HEADERS_URL: Custom URL for remote config (optional)
 """
 
+import asyncio
 import json
 import os
+import threading
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor
+from functools import cache
 from importlib.resources import files
-from typing import Final
+from typing import Final, TypeAlias
 
 import httpx
+from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
 
 # Cache for the loaded configuration
 _BETA_HEADERS_CONFIG: dict | None = None
+_BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock()
+_BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config")
+
+BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
+_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
 
 
 class GetAnthropicBetaHeadersConfig:
@@ -174,13 +185,27 @@ def _load_beta_headers_config() -> dict:
     if _BETA_HEADERS_CONFIG is not None:
         return _BETA_HEADERS_CONFIG
 
-    # Get the URL from environment or use default
+    with _BETA_HEADERS_CONFIG_LOCK:
-    from litellm import anthropic_beta_headers_url
+        if _BETA_HEADERS_CONFIG is not None:
+            return _BETA_HEADERS_CONFIG
 
-    _BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
+        # Get the URL from environment or use default
-    verbose_logger.debug("Loaded and cached beta headers config")
+        from litellm import anthropic_beta_headers_url
 
-    return _BETA_HEADERS_CONFIG
+        _BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
+        verbose_logger.debug("Loaded and cached beta headers config")
+
+        return _BETA_HEADERS_CONFIG
+
+
+async def aload_beta_headers_config() -> None:
+    if _BETA_HEADERS_CONFIG is None:
+        await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config)
+
+
+@cache
+def _bundled_beta_headers_config() -> BetaHeadersConfig:
+    return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config())
 
 
 def reload_beta_headers_config() -> dict:
@@ -288,6 +313,14 @@ def is_beta_header_supported(
     return beta_header in provider_mapping and provider_mapping[beta_header] is not None
 
 
+def is_beta_header_supported_by_bundled_config(beta_header: str, provider: str) -> bool:
+    config: Final = _bundled_beta_headers_config()
+    aliases: Final = config.get("provider_aliases", {})
+    canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
+    provider_mapping: Final = config.get(canonical_provider, {})
+    return isinstance(provider_mapping, Mapping) and provider_mapping.get(beta_header) is not None
+
+
 def get_provider_beta_header(
     anthropic_beta_header: str,
     provider: str,

litellm/assistants/main.py

@@ -129,6 +129,7 @@ def get_assistants(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             aget_assistants=aget_assistants,
         )
@@ -312,6 +313,7 @@ def create_assistants(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             create_assistant_data=create_assistant_data,
             client=client,
             async_create_assistants=async_create_assistants,
@@ -461,6 +463,7 @@ def delete_assistant(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             assistant_id=assistant_id,
             client=client,
             async_delete_assistants=async_delete_assistants,
@@ -637,6 +640,7 @@ def create_thread(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             acreate_thread=acreate_thread,
         )
@@ -783,6 +787,7 @@ def get_thread(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             aget_thread=aget_thread,
         )
@@ -960,6 +965,7 @@ def add_message(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             a_add_message=a_add_message,
         )
@@ -1111,6 +1117,7 @@ def get_messages(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             aget_messages=aget_messages,
         )
@@ -1308,6 +1315,7 @@ def run_thread(
             timeout=timeout,
             max_retries=optional_params.max_retries,
             organization=organization,
+            litellm_params=litellm_params_dict,
             client=client,
             arun_thread=arun_thread,
             event_handler=event_handler,

litellm/batches/batch_line_items.py

@@ -38,6 +38,7 @@ class BatchLineItemSource:
     cost_provider: BatchFileProvider
     model_info: ModelInfo | None
     metadata: StandardBatchLineItemMetadata
+    vertex_location: str | None = None
 
 
 def line_item_metadata(attribution: Mapping[str, object]) -> StandardBatchLineItemMetadata:
@@ -140,7 +141,11 @@ def _line_item(
     source: BatchLineItemSource, custom_id: str | None, request: Mapping[str, object] | None, entry: dict[str, object]
 ) -> StandardBatchLineItem:
     priced: Final = batch_cost_and_usage(
-        (entry,), source.cost_provider, model_name=source.model, model_info=source.model_info
+        (entry,),
+        source.cost_provider,
+        model_name=source.model,
+        model_info=source.model_info,
+        vertex_location=source.vertex_location,
     )
     succeeded: Final = priced.successful_requests > 0
     return StandardBatchLineItem(

litellm/batches/batch_utils.py

@@ -19,7 +19,7 @@ from litellm.llms.vertex_ai.batches.transformation import (
 )
 from litellm.types.llms.openai import Batch
 from litellm.types.utils import ModelInfo, Usage
-from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS
+from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS
 from litellm.utils import token_counter
 
 
@@ -77,9 +77,10 @@ async def calculate_batch_cost_and_usage(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """Calculate the cost and usage of a batch, see batch_cost_and_usage."""
-    return batch_cost_and_usage(file_content_dictionary, custom_llm_provider, model_name, model_info)
+    return batch_cost_and_usage(file_content_dictionary, custom_llm_provider, model_name, model_info, vertex_location)
 
 
 def batch_cost_and_usage(
@@ -87,6 +88,7 @@ def batch_cost_and_usage(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """
     Calculate the cost and usage of a batch.
@@ -96,16 +98,21 @@ def batch_cost_and_usage(
             pricing. Threaded through to batch_cost_calculator so that
             deployment-specific pricing (e.g. input_cost_per_token_batches)
             is used instead of the global cost map.
+        vertex_location: The Vertex AI location the batch ran on; non-global
+            locations apply the model's regional-endpoint uplift multiplier.
     """
     first_row: Final = file_content_dictionary[0] if file_content_dictionary else None
     if _uses_native_vertex_output(custom_llm_provider, model_name, first_row):
-        return calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name, model_info=model_info)
+        return calculate_vertex_ai_batch_cost_and_usage(
+            file_content_dictionary, model_name, model_info=model_info, vertex_location=vertex_location
+        )
 
     return _aggregate_batch_cost_usage_models(
         entries=file_content_dictionary,
         custom_llm_provider=custom_llm_provider,
         model_name=model_name,
         model_info=model_info,
+        vertex_location=vertex_location,
     )
 
 
@@ -115,6 +122,7 @@ async def handle_completed_batch(
     model_name: str | None = None,
     litellm_params: dict[str, object] | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """Fetch a completed batch's output file and aggregate its cost, usage, and
     models in a single pass over the JSONL lines, so the parsed file content is
@@ -128,6 +136,8 @@ async def handle_completed_batch(
         model_info: Optional deployment-level model info with custom pricing,
             threaded through so a deployment's configured rates win over the
             global cost map.
+        vertex_location: The Vertex AI location the batch ran on; non-global
+            locations apply the model's regional-endpoint uplift multiplier.
     """
     # A completed batch whose request lines all failed has no output file - the
     # results are written to a separate error_file_id and output_file_id is None.
@@ -155,7 +165,10 @@ async def handle_completed_batch(
 
     output_file_result: Final = (
         calculate_vertex_ai_batch_cost_and_usage(
-            _iter_batch_output_entries(file_content), model_name, model_info=model_info
+            _iter_batch_output_entries(file_content),
+            model_name,
+            model_info=model_info,
+            vertex_location=vertex_location,
         )
         if _uses_native_vertex_output(
             custom_llm_provider, model_name, next(_iter_batch_output_entries(file_content), None)
@@ -165,6 +178,7 @@ async def handle_completed_batch(
             custom_llm_provider=custom_llm_provider,
             model_name=model_name,
             model_info=model_info,
+            vertex_location=vertex_location,
         )
     )
 
@@ -203,6 +217,7 @@ def _classify_output_line_stats(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> Iterator[_BatchOutputLineStats | _LineOutcome]:
     """Classify every output line in a single pass, so counting failures never needs
     a second read of a potentially huge output file. A line the provider reported as
@@ -213,7 +228,7 @@ def _classify_output_line_stats(
         if not _batch_response_was_successful(entry, custom_llm_provider):
             yield _LineOutcome.PROVIDER_FAILED
             continue
-        stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info)
+        stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location)
         yield stats if stats is not None else _LineOutcome.UNCOSTABLE
 
 
@@ -222,13 +237,14 @@ def _safe_output_line_stats(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> _BatchOutputLineStats | None:
     """Return the stats for one provider-successful batch output line, or None when
     it cannot be costed, so a single bad line never aborts the whole batch's cost
     accounting."""
     custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None
     try:
-        return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info)
+        return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location)
     except Exception as e:  # noqa: BLE001  # any single line's costing failure must not abort the whole batch
         verbose_logger.warning(
             "batch output line could not be costed, so it is billed at $0 and the rest of the batch "
@@ -244,6 +260,7 @@ def _compute_output_line_stats(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> _BatchOutputLineStats:
     response_body: Final = _get_response_from_batch_job_output_file(entry, custom_llm_provider)
     usage: Final = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider)
@@ -258,6 +275,7 @@ def _compute_output_line_stats(
         model_name=model_name,
         response_model=response_model,
         model_info=model_info,
+        vertex_location=vertex_location,
     )
     return _BatchOutputLineStats(
         prompt_cost=line_prompt_cost,
@@ -287,6 +305,7 @@ def _output_line_cost(
     model_name: str | None,
     response_model: str | None,
     model_info: ModelInfo | None,
+    vertex_location: str | None = None,
 ) -> tuple[float, float]:
     """(prompt_cost, completion_cost) for one output line, priced at batch rates."""
     from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost
@@ -307,6 +326,7 @@ def _output_line_cost(
         model=cost_model,
         custom_llm_provider=custom_llm_provider,
         model_info=model_info,
+        vertex_location=vertex_location,
     )
 
 
@@ -315,11 +335,14 @@ def _aggregate_batch_cost_usage_models(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """Aggregate cost, usage, models, and pass/fail counts from batch output
     entries in a single pass, holding one small stats record per line instead
     of the parsed file."""
-    all_results: Final = tuple(_classify_output_line_stats(entries, custom_llm_provider, model_name, model_info))
+    all_results: Final = tuple(
+        _classify_output_line_stats(entries, custom_llm_provider, model_name, model_info, vertex_location)
+    )
     line_stats: Final = tuple(result for result in all_results if isinstance(result, _BatchOutputLineStats))
     failed_requests: Final = sum(1 for result in all_results if result is _LineOutcome.PROVIDER_FAILED)
     successful_requests: Final = len(all_results) - failed_requests
@@ -366,6 +389,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
     vertex_ai_batch_responses: Iterable[dict[str, object]],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
+    vertex_location: str | None = None,
 ) -> BatchCostUsageResult:
     """
     Cost and usage of a native Vertex predictions.jsonl, one
@@ -384,6 +408,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
             model_info=model_info,
             calculate_usage=VertexGeminiConfig.calculate_usage,
             cost_calculator=batch_cost_calculator,
+            vertex_location=vertex_location,
         )
         for row in vertex_ai_batch_responses
     )
@@ -569,6 +594,7 @@ def extract_file_access_credentials(
             # A federated deployment holds no api_key, so without these the fetch that reads a
             # finished batch's output has nothing to authenticate with and its cost is never billed.
             *sorted(ANTHROPIC_WIF_KWARGS_KEYS),
+            *sorted(OPENAI_WIF_KWARGS_KEYS),
         )
         for key in credential_keys:
             if key in litellm_params:
@@ -793,11 +819,17 @@ def _batch_response_was_successful(
     message batch results lines report ``result.type == "succeeded"``; Bedrock
     batch output lines report ``modelOutput`` (and no ``error``).
     """
-    if custom_llm_provider == "anthropic":
-        return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
     if custom_llm_provider == "bedrock":
         return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None
-    _response: Final[dict[str, object]] = cast(  # cast-ok: batch response bodies come from provider JSON
+    envelope_key: Final = "result" if custom_llm_provider == "anthropic" else "response"
-        dict[str, object], batch_job_output_file.get("response", None) or {}
+    envelope: Final = batch_job_output_file.get(envelope_key)
+    if not isinstance(envelope, Mapping):
+        if envelope is not None:
+            verbose_logger.warning("batch output %s must be an object, treating the line as unsuccessful", envelope_key)
+        return False
+    response: Final = cast(Mapping[object, object], envelope)  # cast-ok: mapping shape validated above
+    return (
+        response.get("type") == "succeeded"
+        if custom_llm_provider == "anthropic"
+        else response.get("status_code") == 200
     )
-    return _response.get("status_code", None) == 200

litellm/batches/main.py

@@ -309,6 +309,7 @@ def create_batch(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
+                litellm_params=litellm_params,
             )
         elif custom_llm_provider == "azure":
             api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@@ -471,6 +472,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
             organization=organization,
             timeout=timeout,
             max_retries=optional_params.max_retries,
+            litellm_params=litellm_params,
         )
     elif custom_llm_provider == "azure":
         api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@@ -819,6 +821,7 @@ def list_batches(
                 organization=organization,
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
+                litellm_params=litellm_params,
             )
         elif custom_llm_provider == "azure":
             api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@@ -1016,6 +1019,7 @@ def cancel_batch(
                 organization=organization,
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
+                litellm_params=litellm_params,
             )
         elif custom_llm_provider == "azure":
             api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")

litellm/caching/caching_handler.py

@@ -346,6 +346,7 @@ class LLMCachingHandler:
 
             if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function):
                 verbose_logger.debug("Checking Async Cache")
+                logging_obj.reset_caching_details()
                 cached_result = await self._retrieve_from_cache(
                     call_type=call_type,
                     kwargs=kwargs,
@@ -357,7 +358,7 @@ class LLMCachingHandler:
                     verbose_logger.debug("Cache Hit!")
                     cache_hit: Final = True
                     end_time: Final = datetime.datetime.now()
-                    model, custom_llm_provider, _, _ = litellm.get_llm_provider(
+                    provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider(
                         model=model,
                         custom_llm_provider=kwargs.get("custom_llm_provider", None),
                         api_base=kwargs.get("api_base", None),
@@ -366,7 +367,7 @@ class LLMCachingHandler:
                     cache_duration_ms: Final = (cache_check_end_time - cache_check_start_time) * 1000
                     self._update_litellm_logging_obj_environment(
                         logging_obj=logging_obj,
-                        model=model,
+                        model=provider_model,
                         kwargs=kwargs,
                         cached_result=cached_result,
                         is_async=True,
@@ -381,7 +382,7 @@ class LLMCachingHandler:
                         call_type=call_type,
                         kwargs=kwargs,
                         logging_obj=logging_obj,
-                        model=model,
+                        model=provider_model,
                         custom_llm_provider=kwargs.get("custom_llm_provider", None),
                         args=args,
                     )
@@ -467,6 +468,7 @@ class LLMCachingHandler:
                 new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
             self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
             print_verbose("Checking Sync Cache")
+            logging_obj.reset_caching_details()
             with response_cache_phase("get"):
                 cached_result = litellm.cache.get_cache(**new_kwargs)
             if is_response_without_output(cached_result):
@@ -514,7 +516,7 @@ class LLMCachingHandler:
                     if not is_internal_call.get() and not _should_defer_streaming_cache_hit_callbacks(
                         cached_result=cached_result
                     ):
-                        logging_obj.handle_sync_success_callbacks_for_async_calls(
+                        logging_obj.handle_sync_cache_hit_success_callbacks(
                             result=cached_result,
                             start_time=start_time,
                             end_time=end_time,

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -41,7 +41,11 @@ from litellm.responses.sse_output_recovery import (
     record_output_item_chunk,
     record_output_text_chunk,
 )
-from litellm.responses.utils import ResponsesAPIRequestUtils, normalize_responses_api_stream_options
+from litellm.responses.utils import (
+    ResponsesAPIRequestUtils,
+    auto_reasoning_summary,
+    normalize_responses_api_stream_options,
+)
 from litellm.types.llms.openai import (
     ChatCompletionAnnotation,
     ChatCompletionReasoningItem,
@@ -176,6 +180,16 @@ def _without_audio_input_parts(input_items: list[object]) -> list[object]:
     return [_without_audio_input_parts_in_item(item) for item in input_items]
 
 
+def _effort_with_requested_summary(nested_effort: object, top_level_effort: object) -> object:
+    match top_level_effort, nested_effort:
+        case {"summary": str() as summary}, str() as effort:
+            return {"effort": effort, "summary": summary}
+        case {"summary": str() as summary}, Mapping() as reasoning if "summary" not in reasoning:
+            return {**reasoning, "summary": summary}
+        case _:
+            return nested_effort
+
+
 def _served_provider(litellm_params: Mapping[str, object]) -> str | None:
     custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
     return custom_llm_provider if isinstance(custom_llm_provider, str) else None
@@ -197,7 +211,7 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool
     return not _supports_audio_input(model, litellm_params)
 
 
-def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
+def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
     return MappingProxyType(
         {
             key: value
@@ -953,6 +967,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         optional_params: dict,
         responses_api_request: "ResponsesAPIOptionalRequestParams",
         drop_params: bool = False,
+        model: str | None = None,
+        custom_llm_provider: str | None = None,
     ) -> None:
         """Map optional_params into responses_api_request (mutates in place)."""
         for key, value in optional_params.items():
@@ -984,7 +1000,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             elif key == "previous_response_id":
                 responses_api_request["previous_response_id"] = value
             elif key == "reasoning_effort":
-                responses_api_request["reasoning"] = self.map_reasoning_effort(value)
+                responses_api_request["reasoning"] = self.map_reasoning_effort(value, model, custom_llm_provider)
             elif key == "web_search_options":
                 self._add_web_search_tool(responses_api_request, value)
 
@@ -1100,6 +1116,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             optional_params,
             responses_api_request,
             drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
+            model=model,
+            custom_llm_provider=_served_provider(litellm_params),
         )
 
         stream: Final = optional_params.get("stream") or litellm_params.get("stream", False)
@@ -1455,7 +1473,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         )
 
         model_response.id = _upstream_response_id(raw_response.id) or raw_response.id
-        for key, value in _provider_metadata(raw_response.model_extra).items():
+        for key, value in responses_provider_metadata(raw_response.model_extra).items():
             setattr(model_response, key, value)
 
         # Preserve hidden params from the ResponsesAPIResponse, especially the headers
@@ -1705,14 +1723,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         # Extract supported params from extra_body and merge into optional_params
         extra_body_copy: Final = extra_body.copy()
+        top_level_effort: Final = cast(object, optional_params.get("reasoning_effort"))  # cast-ok: upcast
         for key, value in extra_body_copy.items():
             if key in supported_responses_api_params:
                 # Prefer extra_body value if it exists (may have more complete info like summary in reasoning_effort)
-                optional_params[key] = extra_body.pop(key)
+                extra_body.pop(key)
+                optional_params[key] = (
+                    _effort_with_requested_summary(cast(object, value), top_level_effort)  # cast-ok: upcast
+                    if key == "reasoning_effort"
+                    else value
+                )
 
         return optional_params
 
-    def map_reasoning_effort(self, reasoning_effort: object) -> Reasoning:
+    def map_reasoning_effort(
+        self, reasoning_effort: object, model: str | None = None, custom_llm_provider: str | None = None
+    ) -> Reasoning:
         # If dict is passed, convert it directly to Reasoning object
         if isinstance(reasoning_effort, dict):
             return Reasoning(
@@ -1726,7 +1752,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         )
 
         return (
-            Reasoning(effort=reasoning_effort, summary="detailed")
+            Reasoning(
+                effort=reasoning_effort,
+                summary="detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider),
+            )
             if auto_summary_enabled
             else Reasoning(effort=reasoning_effort)
         )
@@ -2139,7 +2168,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                 from litellm.responses.utils import ResponseAPILoggingUtils
 
                 usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_data.get("usage"))
-            provider_metadata: Final = _provider_metadata(response_data)
+            provider_metadata: Final = responses_provider_metadata(response_data)
             served_service_tier: Final = response_data.get("service_tier")
             return ModelResponseStream(
                 choices=[

litellm/constants.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 from enum import Enum
 from types import MappingProxyType
@@ -18,6 +19,9 @@ MICROSOFT_365_COPILOT_DEFAULT_TOKEN_EXCHANGE_PROFILE: Final = "jwt_bearer_obo"
 MICROSOFT_365_COPILOT_DEFAULT_TOKEN_EXCHANGE_SCOPE: Final = "https://graph.microsoft.com/.default"
 # Replies depend on the caller's delegated identity, which response-cache keys do not include.
 RESPONSE_CACHE_EXCLUDED_PROVIDERS: Final = frozenset({"microsoft_365_copilot", "databricks_agent"})
+CLAUDE_CODE_PICKER_PATTERN: Final = re.compile(r"claude|anthropic", re.IGNORECASE)
+GATEWAY_CLIENT_HEADER: Final = "x-gateway-client"
+CLAUDE_CODE_CLIENT: Final = "claude-code"
 MICROSOFT_365_COPILOT_DEFAULT_TIME_ZONE: Final = "UTC"
 GITHUB_COPILOT_PER_USER_AUTH_TYPE: Final = "per_user_oauth"
 GITHUB_COPILOT_AUTH_TYPE_KEY: Final = "github_copilot_auth_type"
@@ -36,6 +40,14 @@ class ServerStreamingClassification(str, Enum):
 SERVER_STREAMING_CLASSIFICATION_MARKER: Final = ServerStreamingClassification.MARKER
 DEFER_PYDANTIC_BUILD: Final = os.getenv("DEFER_PYDANTIC_BUILD", "true") in ("true", "1", "on")
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
+KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS: Final = get_env_float(
+    "KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS", 5.0
+)
+KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS: Final = get_env_float(
+    "KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS", 300.0
+)
+KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS: Final = 2.0
+KUBERNETES_POD_ROUTING_KEY: Final = "kubernetes_pod_routing"
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
 AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"})
 ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5)
@@ -179,6 +191,11 @@ DEFAULT_IMAGE_HEIGHT: Final = get_env_int("DEFAULT_IMAGE_HEIGHT", 300)
 # Maps to OpenAI's 50 MB payload limit - requests with images exceeding this size will be rejected
 # Set MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0 to disable image URL handling entirely
 MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: Final = get_env_float("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 50.0)
+MAX_DECOMPRESSED_REQUEST_BODY_SIZE_MB: Final = get_env_float("MAX_DECOMPRESSED_REQUEST_BODY_SIZE_MB", 50.0)
+MAX_REQUEST_BODY_DECOMPRESSION_RATIO: Final = get_env_int("MAX_REQUEST_BODY_DECOMPRESSION_RATIO", 200)
+MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB: Final = get_env_float(
+    "MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB", 200.0
+)
 MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB: Final = get_env_int(
     "MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024
 )  # 1MB = 1024KB
@@ -240,6 +257,13 @@ MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_PER_USER_TOKE
 # MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers.
 MCP_CLIENT_TIMEOUT: Final = get_env_float("LITELLM_MCP_CLIENT_TIMEOUT", 60.0)
 MCP_TOOL_LISTING_TIMEOUT: Final = get_env_float("LITELLM_MCP_TOOL_LISTING_TIMEOUT", 30.0)
+MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES: Final = get_env_int("LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES", 3)
+MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS: Final = get_env_float(
+    "LITELLM_MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS", 0.5
+)
+MCP_UPSTREAM_RATE_LIMIT_MAX_DELAY_SECONDS: Final = get_env_float(
+    "LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_DELAY_SECONDS", 5.0
+)
 MCP_METADATA_TIMEOUT: Final = get_env_float("LITELLM_MCP_METADATA_TIMEOUT", 10.0)
 MCP_HEALTH_CHECK_TIMEOUT: Final = get_env_float("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", 10.0)
 AGENT_HEALTH_CHECK_TIMEOUT_SECONDS: Final = get_env_float("LITELLM_AGENT_HEALTH_CHECK_TIMEOUT", 5.0)
@@ -391,7 +415,7 @@ DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
 NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
 DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
 
-BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
+BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
 BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
 BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
 BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime"
@@ -1026,10 +1050,11 @@ openai_compatible_endpoints: Final[list[str]] = [
     "https://api.prisminference.com/v1",
     "https://api.reka.ai/v1",
     "https://gigachat.devices.sberbank.ru/api/v1",
+    "https://api.openinfer.ai/v1",
 ]
 
 
-openai_compatible_providers: Final[list] = [
+openai_compatible_providers: Final[list[str]] = [
     "anyscale",
     "groq",
     "nvidia_nim",
@@ -1094,6 +1119,7 @@ openai_compatible_providers: Final[list] = [
     "docker_model_runner",
     "ragflow",
     "pinstripes",  # Pinstripes - JSON-configured provider
+    "openinfer",
     "darkbloom",
     "meta",  # Meta Model API (Muse Spark) - JSON-configured provider
     "cognition",
@@ -1250,14 +1276,6 @@ together_ai_models: Final[set] = set(
 # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
 
 
-baseten_models: Final[set] = set(
-    [
-        "qvv0xeq",
-        "q841o8w",
-        "31dxrj3",
-    ]
-)  # FALCON 7B  # WizardLM  # Mosaic ML
-
 featherless_ai_models: Final[set] = set(
     [
         "featherless-ai/Qwerky-72B",
@@ -1615,6 +1633,7 @@ CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = get_env_int("CLOUDZERO_EXPORT_INTERVA
 MCP_TOOL_NAME_PREFIX: Final = "mcp_tool"
 MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = get_env_int("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100)
 PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096
+INTERNAL_CALLBACK_NAMES: Final = frozenset({"cache", "vector_store_pre_call_hook"})
 
 # Headers to control callbacks
 X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
@@ -1626,6 +1645,7 @@ OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_to
 CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
+VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
 ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"

litellm/containers/README.md

@@ -120,6 +120,7 @@ class AzureContainerConfig(BaseContainerConfig):
         self,
         headers: dict,
         api_key: Optional[str] = None,
+        litellm_params: Optional[GenericLiteLLMParams] = None,
     ) -> dict:
         """Azure uses api-key header instead of Bearer token."""
         import litellm

litellm/cost_calculator.py

@@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence
 from datetime import datetime
 from functools import lru_cache
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
 
 from httpx import Response
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@@ -45,6 +45,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
     get_regional_uplift_multiplier,
     get_service_tier_cost_key,
     get_token_type_cost_breakdown,
+    get_vertex_regional_endpoint_uplift,
     parse_prompt_tokens_details,
     select_cost_metric_for_model,
 )
@@ -137,6 +138,7 @@ from litellm.types.utils import (
     TranscriptionUsageTokensObject,
     Usage,
     VectorStoreSearchResponse,
+    is_server_derived_pricing_key,
 )
 from litellm.utils import (
     CallTypes,
@@ -149,7 +151,10 @@ from litellm.utils import (
     TextCompletionResponse,
     TranscriptionResponse,
     cached_get_model_info_helper,
+    model_info_with_pricing_overrides,
+    runtime_model_cost_registrations,
     token_counter,
+    unmapped_model_generalization,
 )
 
 if TYPE_CHECKING:
@@ -436,6 +441,7 @@ def cost_per_token(
     ### REQUEST MODEL ###
     request_model: str | None = None,  # original request model for router detection
     custom_model_info: OCRPricing | None = None,
+    bedrock_model_info: ModelInfo | None = None,
 ) -> tuple[float, float]:
     """
     Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -675,6 +681,7 @@ def cost_per_token(
             model=model,
             custom_llm_provider=custom_llm_provider,
             data_residency=data_residency,
+            vertex_location=vertex_location,
         )
     elif call_type == "atranscription" or call_type == "transcription":
         if _transcription_usage_has_token_details(usage_block):
@@ -737,6 +744,15 @@ def cost_per_token(
     elif custom_llm_provider == "anthropic":
         return anthropic_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
     elif custom_llm_provider == "bedrock":
+        resolved_bedrock_info: Final = bedrock_model_info or _bedrock_pricing_info(model, None, "bedrock", False)
+        if resolved_bedrock_info is not None:
+            return generic_cost_per_token(
+                model=model,
+                usage=usage_block,
+                custom_llm_provider="bedrock",
+                service_tier=service_tier,
+                model_info=resolved_bedrock_info,
+            )
         return bedrock_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
     elif custom_llm_provider == "openai":
         return openai_cost_per_token(
@@ -859,6 +875,37 @@ def _cost_map_entry_prices_anything(entry: Mapping[str, object]) -> bool:
     )
 
 
+def _bedrock_registration_has_prices(entry: Mapping[str, object]) -> bool:
+    return _cost_map_entry_prices_anything(entry) or any(
+        value is not None and field.startswith("cache_") and is_server_derived_pricing_key(field)
+        for field, value in entry.items()
+    )
+
+
+def _bedrock_pricing_info(
+    model: str | None,
+    invoked_model_id: str | None,
+    custom_llm_provider: str | None,
+    explicit_pricing: bool,
+) -> ModelInfo | None:
+    if explicit_pricing or custom_llm_provider != "bedrock" or not model:
+        return None
+    registration: Final = next(
+        (
+            (key, entry)
+            for key, entry in runtime_model_cost_registrations(model, "bedrock")
+            if _bedrock_registration_has_prices(entry)
+        ),
+        None,
+    )
+    profile: Final = _cost_map_model_info(invoked_model_id, "bedrock") if invoked_model_id else None
+    if profile is None or not _declares_rates(profile):
+        return _cost_map_model_info(registration[0], None) if registration is not None else None
+    if registration is None or registration[0] == profile["key"]:
+        return profile
+    return model_info_with_pricing_overrides(profile, registration[1])
+
+
 def _declares_rates(model_info: ModelInfo | None) -> bool:
     entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None
     return entry is not None and _cost_map_entry_prices_anything(entry)
@@ -875,6 +922,26 @@ def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | Non
     return own_model_info["litellm_provider"]
 
 
+def _pricing_provider_for_cost_candidate(
+    candidate_provider: str | None,
+    request_provider: str | None,
+    current_provider: object,
+) -> str | None:
+    if candidate_provider != request_provider:
+        return candidate_provider
+    return current_provider if isinstance(current_provider, str) else None
+
+
+def _invoked_model_id_for_cost(
+    optional_params: object,
+    logging_obj: LitellmLoggingObject | None,
+) -> str | None:
+    recorded: Final[object] = getattr(logging_obj, "model_call_details", None)
+    return _get_hidden_str_for_cost_calc(optional_params, "model_id") or _get_hidden_str_for_cost_calc(
+        recorded, "model_id"
+    )
+
+
 def select_model_name_for_cost_calc(
     model: str | None,
     completion_response: object | None,
@@ -883,6 +950,7 @@ def select_model_name_for_cost_calc(
     custom_llm_provider: str | None = None,
     router_model_id: str | None = None,
     region_name: str | None = None,
+    invoked_model_id: str | None = None,
 ) -> str | None:
     """
     1. If custom pricing is true, return received model name
@@ -903,6 +971,13 @@ def select_model_name_for_cost_calc(
     hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
     provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
     explicit_pricing: Final = custom_pricing is True or base_model is not None
+    configured_model: Final = model or (
+        completion_response_model if isinstance(completion_response_model, str) else None
+    )
+    bedrock_model_info: Final = _bedrock_pricing_info(
+        configured_model, invoked_model_id, custom_llm_provider, explicit_pricing
+    )
+    bedrock_pricing_model: Final = bedrock_model_info["key"] if bedrock_model_info is not None else None
     priced_from_response: Final = provider_response_model is not None or completion_response_model is not None
     priced_region: Final = (
         _get_hidden_str_for_cost_calc(hidden_params, "region_name") or region_name
@@ -920,8 +995,8 @@ def select_model_name_for_cost_calc(
         else:
             return_model = model
 
-    elif base_model is not None or provider_response_model is not None:
+    elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None:
-        return_model = base_model if base_model is not None else provider_response_model
+        return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model
 
     elif completion_response_model is None and hidden_params is not None:
         if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
@@ -1092,6 +1167,8 @@ def get_usage_object(
 
     if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
         return None if completion_response.usage is None else _decisions_usage(completion_response.usage)
+    if isinstance(completion_response, HttpxBinaryResponseContent):
+        return completion_response.usage
     if usage_obj is None:
         return None
     if isinstance(usage_obj, Usage):
@@ -1438,32 +1515,15 @@ def completion_cost(
     vertex_location: str | None = None,  # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
 ) -> float:
     """
-    Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
+    Return the USD cost of a LiteLLM call.
 
-    Parameters:
+    Pass `completion_response` to price a response LiteLLM returned. Without one, pass `model` plus
-        completion_response (litellm.ModelResponses): [Required] The response received from a LiteLLM completion request.
+    `messages` / `prompt` and `completion`, and token counts are computed from the text. Prices come from
+    `litellm.model_cost` unless `custom_cost_per_token` or `custom_cost_per_second` is given. `total_time`
+    is in milliseconds and only matters for per-second pricing such as Replicate and SageMaker.
 
-        [OPTIONAL PARAMS]
+    Raises if no candidate model name resolves to pricing. Register the model via `litellm.register_model`
-        model (str): Optional. The name of the language model used in the completion calls
+    or add it to model_prices_and_context_window.json.
-        prompt (str): Optional. The input prompt passed to the llm
-        completion (str): Optional. The output completion text from the llm
-        total_time (float, int): Optional. (Only used for Replicate LLMs) The total time used for the request in seconds
-        custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call.
-        custom_cost_per_second: Optional[float]: the cost per second for the llm api call.
-
-    Returns:
-        float: The cost in USD dollars for the completion based on the provided parameters.
-
-    Exceptions:
-        Raises exception if model not in the litellm model cost map. Register model, via custom pricing or PR - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json
-
-
-    Note:
-        - If completion_response is provided, the function extracts token information and the model name from it.
-        - If completion_response is not provided, the function calculates token counts based on the model and input text.
-        - The cost is calculated based on the model, prompt tokens, and completion tokens.
-        - For certain models containing "togethercomputer" in the name, prices are based on the model size.
-        - For un-mapped Replicate models, the cost is calculated based on the total time used for the request.
     """
     try:
         call_type = infer_call_type(call_type, completion_response) or "completion"
@@ -1550,6 +1610,10 @@ def completion_cost(
             if base_model is not None and custom_pricing is not True
             else None
         )
+        invoked_model_id: Final = _invoked_model_id_for_cost(
+            _LOGGED_LITELLM_PARAMS.validate_python(optional_params) if optional_params is not None else None,
+            litellm_logging_obj,
+        )
         selected_model: Final = select_model_name_for_cost_calc(
             model=model,
             completion_response=completion_response,
@@ -1558,17 +1622,30 @@ def completion_cost(
             base_model=base_model,
             router_model_id=router_model_id,
             region_name=region_name,
+            invoked_model_id=invoked_model_id,
         )
 
+        bedrock_model_info: Final = _bedrock_pricing_info(
+            model or _get_response_model(completion_response),
+            invoked_model_id,
+            _get_provider_for_cost_calc(model, custom_llm_provider),
+            explicit_pricing,
+        )
         requested_model: Final = model
-        potential_model_names: Final = [
+        request_pricing_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
-            selected_model,
+        raw_candidates: Final = (
-            _get_response_model(completion_response),
+            (
-        ]
+                selected_model,
-        if model is not None:
+                base_model_provider if base_model_provider is not None else request_pricing_provider,
-            potential_model_names.append(model)
+            ),
+            (_get_response_model(completion_response), request_pricing_provider),
+            *(((model, request_pricing_provider),) if model is not None else ()),
+        )
+        potential_model_names: Final = (
+            raw_candidates if explicit_pricing else _priced_candidates_before_unpriced_rules(raw_candidates)
+        )
 
-        for idx, model in enumerate(potential_model_names):
+        for idx, (model, pricing_provider) in enumerate(potential_model_names):
             try:
                 if verbose_logger.isEnabledFor(logging.DEBUG):
                     verbose_logger.debug("selected model name for cost calculation: %s", model)
@@ -1913,13 +1990,6 @@ def completion_cost(
                 if litellm_logging_obj is not None:
                     request_model_for_cost = litellm_logging_obj.model
 
-                pricing_provider: str | None = (
-                    base_model_provider
-                    if base_model_provider is not None and idx == 0
-                    else custom_llm_provider
-                    if isinstance(custom_llm_provider, str)
-                    else None
-                )
                 (
                     prompt_tokens_cost_usd_dollar,
                     completion_tokens_cost_usd_dollar,
@@ -1927,7 +1997,11 @@ def completion_cost(
                     model=model,
                     prompt_tokens=prompt_tokens or 0,
                     completion_tokens=completion_tokens or 0,
-                    custom_llm_provider=pricing_provider,
+                    custom_llm_provider=_pricing_provider_for_cost_candidate(
+                        candidate_provider=pricing_provider,
+                        request_provider=request_pricing_provider,
+                        current_provider=custom_llm_provider,
+                    ),
                     response_time_ms=response_time_ms,
                     region_name=None if explicit_pricing else region_name,
                     custom_cost_per_second=custom_cost_per_second,
@@ -1945,6 +2019,7 @@ def completion_cost(
                     vertex_location=vertex_location,
                     response=completion_response,
                     custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id),
+                    bedrock_model_info=bedrock_model_info,
                 )
 
                 # Get additional costs from provider (e.g., routing fees, infrastructure costs)
@@ -2026,12 +2101,17 @@ def completion_cost(
                     if cost_per_token_usage_object is not None and model:
                         _token_type_breakdown = get_token_type_cost_breakdown(
                             model=model,
-                            custom_llm_provider=pricing_provider,
+                            custom_llm_provider=_pricing_provider_for_cost_candidate(
+                                candidate_provider=pricing_provider,
+                                request_provider=request_pricing_provider,
+                                current_provider=custom_llm_provider,
+                            ),
                             usage=cost_per_token_usage_object,
                             service_tier=service_tier,
                             data_residency=data_residency,
                             vertex_location=vertex_location,
                             custom_cost_per_token=custom_cost_per_token,
+                            model_info=bedrock_model_info,
                         )
                         _reasoning_cost = _token_type_breakdown.reasoning_cost
                         _cache_read_cost = _token_type_breakdown.cache_read_cost
@@ -2066,7 +2146,7 @@ def completion_cost(
                 )
                 if idx == len(potential_model_names) - 1:
                     raise e
-        raise Exception(f"Unable to calculat cost for received potential model names - {potential_model_names}")
+        raise Exception(f"Unable to calculate cost for received potential model names - {potential_model_names}")
     except Exception as e:
         raise e
 
@@ -2081,13 +2161,20 @@ def get_response_cost_from_hidden_params(
 
     additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
     if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
-        response_cost: Final = additional_headers["llm_provider-x-litellm-response-cost"]
+        return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
-        if response_cost is None:
-            return None
-        return float(additional_headers["llm_provider-x-litellm-response-cost"])
     return None
 
 
+def _billable_header_cost(value: object) -> float | None:
+    if not isinstance(value, (str, int, float)):
+        return None
+    try:
+        cost: Final = float(value)
+    except ValueError:
+        return None
+    return cost if math.isfinite(cost) and cost >= 0 else None
+
+
 def response_cost_calculator(
     response_object: ModelResponse
     | EmbeddingResponse
@@ -2281,6 +2368,25 @@ def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelIn
         return None
 
 
+_CostCandidate: TypeAlias = tuple[str | None, str | None]
+
+
+def _is_unpriced_generalized_candidate(candidate: _CostCandidate) -> bool:
+    model, pricing_provider = candidate
+    if model is None:
+        return False
+    rule_info: Final = unmapped_model_generalization(model, pricing_provider)
+    return rule_info is not None and not _cost_map_entry_prices_anything(rule_info)
+
+
+def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, ...]) -> tuple[_CostCandidate, ...]:
+    unpriced: Final = tuple(_is_unpriced_generalized_candidate(candidate) for candidate in candidates)
+    return (
+        *(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if not is_unpriced),
+        *(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if is_unpriced),
+    )
+
+
 def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
     raw_entry: Final = litellm.model_cost.get(key)
     return raw_entry if isinstance(raw_entry, Mapping) else None
@@ -2308,7 +2414,16 @@ def pricing_entry_for_cost_calc(
         custom_llm_provider=custom_llm_provider,
         router_model_id=router_model_id,
         region_name=region_name,
+        invoked_model_id=_invoked_model_id_for_cost(None, litellm_logging_obj),
     )
+    bedrock_model_info: Final = _bedrock_pricing_info(
+        model or _get_response_model(completion_response),
+        _invoked_model_id_for_cost(None, litellm_logging_obj),
+        _get_provider_for_cost_calc(model, custom_llm_provider),
+        custom_pricing is True or base_model is not None,
+    )
+    if bedrock_model_info is not None:
+        return bedrock_model_info["key"], bedrock_model_info
     candidates: Final = (selected_model, _get_response_model(completion_response), model)
     resolved: Final = next(
         (info for info in (_cost_map_model_info(name, custom_llm_provider) for name in candidates if name) if info),
@@ -2756,6 +2871,8 @@ def batch_cost_calculator(
     custom_llm_provider: str | None = None,
     model_info: ModelInfo | None = None,
     data_residency: str | None = None,
+    *,
+    vertex_location: str | None = None,
 ) -> tuple[float, float]:
     """
     Calculate the cost of a batch job.
@@ -2845,7 +2962,8 @@ def batch_cost_calculator(
         total_prompt_cost *= uplift
         total_completion_cost *= uplift
 
-    return total_prompt_cost, total_completion_cost
+    vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
+    return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift
 
 
 def _batch_prompt_cost(
@@ -3086,14 +3204,19 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
         return response_ids[-1] if response_ids else None
 
     @staticmethod
-    def create_logging_object(results: Sequence[Mapping[str, object]]) -> LiteLLMRealtimeStreamLoggingObject:
+    def create_logging_object(
+        results: Sequence[Mapping[str, object]], litellm_model_name: str | None = None
+    ) -> LiteLLMRealtimeStreamLoggingObject:
         tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(results)
-        return LiteLLMRealtimeStreamLoggingObject(
+        logging_object: Final = LiteLLMRealtimeStreamLoggingObject(
             id=ResponsesWebSocketTokenUsageProcessor.response_id_from_responses_ws_results(results),
             usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(results),
             results=cast(OpenAIRealtimeStreamList, list(results)),  # cast-ok: raw WS frames, unvalidated field
             service_tier=next(iter(tier_partition)) if len(tier_partition) == 1 else None,
         )
+        if litellm_model_name is not None:
+            logging_object.hidden_params = {"litellm_model_name": litellm_model_name}
+        return logging_object
 
 
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"

litellm/decisions/call.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from functools import partial
@@ -6,6 +7,7 @@ from typing import Final, Literal, TypeAlias
 
 import httpx
 from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
+from pydantic_core import to_jsonable_python
 from typing_extensions import assert_never
 
 import litellm
@@ -56,8 +58,9 @@ from litellm.types.decisions import (
     OpenAIDecisionResponse,
     UnsupportedDecisionsRequest,
 )
+from litellm.types.llms.openai import ChatCompletionUserMessage
 from litellm.types.utils import LlmProviders, ProviderSpecificHeader
-from litellm.utils import ProviderConfigManager
+from litellm.utils import ProviderConfigManager, get_model_cost_key
 
 SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
 OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
@@ -415,6 +418,30 @@ def _outbound_headers(
     return MappingProxyType({**(extra_headers or {}), **forwarded})
 
 
+_LOGGED_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+
+
+def _jsonable_fallback(value: object) -> object:
+    return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value)
+
+
+def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage:
+    caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python(
+        to_jsonable_python(caller_request, fallback=_jsonable_fallback)
+    )
+    sent_fields: Final = json.dumps(
+        dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {}))
+    )
+    return ChatCompletionUserMessage(role="user", content=sent_fields)
+
+
+def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None:
+    """Record every request field the caller sent as the logged `messages`, before any validation."""
+    logging_obj: Final = kwargs.get("litellm_logging_obj")
+    if isinstance(logging_obj, LiteLLMLoggingObj):
+        logging_obj.update_messages([_logged_request_message(caller_request)])
+
+
 def prepare_call(
     *,
     model: str,
@@ -458,7 +485,7 @@ def prepare_call(
         ) from error
 
     try:
-        resolved_api_base, resolved_api_key = provider_config.resolve_connection(
+        resolved_api_base, static_api_key = provider_config.resolve_connection(
             dynamic_api_base or api_base, dynamic_api_key or api_key
         )
     except ValueError as error:
@@ -469,6 +496,7 @@ def prepare_call(
             model=model,
             llm_provider=provider,
         )
+    resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs)
     if resolved_api_key is None and provider_config.api_key_required:
         raise litellm.AuthenticationError(
             message=f"Missing API key for Decisions provider '{provider}'",
@@ -666,7 +694,10 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
         {
             "model": f"{call.custom_llm_provider}/{call.model}",
             "custom_llm_provider": call.custom_llm_provider,
-            "provider_response_model": f"{call.custom_llm_provider}/{call.model}",
+            "provider_response_model": (
+                get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
+                or f"{call.custom_llm_provider}/{call.model}"
+            ),
             **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
         }
     )

litellm/decisions/main.py

@@ -1,5 +1,7 @@
+import asyncio
 import warnings
 from collections.abc import Mapping
+from functools import partial
 from typing import Final
 
 import httpx
@@ -15,6 +17,7 @@ from litellm.decisions.call import (
     SystemOneQuestions,
     SystemOneRequestFields,
     asend,
+    log_caller_request,
     prepare_call,
     reject_images_next_to_input,
     send,
@@ -76,6 +79,16 @@ def _prepare_decisions_call(
     extra_headers: Mapping[str, str] | None,
     kwargs: Mapping[str, object],
 ) -> DecisionsCall:
+    log_caller_request(
+        kwargs,
+        {
+            "input": decision_input,
+            "questions": questions,
+            "audio": audio,
+            "videos": videos,
+            **{key: kwargs[key] for key in _SYSTEMONE_FIELDS if key in kwargs},
+        },
+    )
     return prepare_call(
         model=model,
         fields=_request_fields(model, decision_input, questions, audio, videos, kwargs),
@@ -104,19 +117,22 @@ async def adecisions(
     videos: DecisionsVideoClips | None = None,
     **kwargs: object,
 ) -> OpenAIDecisionResponse | DecisionsResponse:
-    call: Final = _prepare_decisions_call(
+    call: Final = await asyncio.to_thread(
-        model=model,
+        partial(
-        decision_input=input,
+            _prepare_decisions_call,
-        questions=questions,
+            model=model,
-        safety_identifier=safety_identifier,
+            decision_input=input,
-        audio=audio,
+            questions=questions,
-        videos=videos,
+            safety_identifier=safety_identifier,
-        api_key=api_key,
+            audio=audio,
-        api_base=api_base,
+            videos=videos,
-        timeout=timeout,
+            api_key=api_key,
-        custom_llm_provider=custom_llm_provider,
+            api_base=api_base,
-        extra_headers=extra_headers,
+            timeout=timeout,
-        kwargs=kwargs,
+            custom_llm_provider=custom_llm_provider,
+            extra_headers=extra_headers,
+            kwargs=kwargs,
+        )
     )
     return await asend(call)
 

litellm/endpoints/speech/speech_to_completion_bridge/handler.py

@@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, Final
 
 from typing_extensions import TypedDict
 
+from litellm._internal_context import internal_sub_call
+
 if TYPE_CHECKING:
     from litellm import LiteLLMLoggingObj
     from litellm.types.llms.openai import HttpxBinaryResponseContent
@@ -111,9 +113,10 @@ class SpeechToCompletionBridgeHandler:
             voice=voice,
         )
 
-        result: Final = completion(
+        with internal_sub_call():
-            **request_data,
+            result: Final = completion(
-        )
+                **request_data,
+            )
 
         requested_response_format: Final = optional_params.get("response_format")
         if isinstance(result, ModelResponse):

litellm/endpoints/speech/speech_to_completion_bridge/transformation.py

@@ -5,11 +5,12 @@ from typing import TYPE_CHECKING, Final, cast
 from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
+from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_param
 
 if TYPE_CHECKING:
     from litellm import Logging as LiteLLMLoggingObj
     from litellm.types.llms.openai import ChatCompletionUserMessage, HttpxBinaryResponseContent
-    from litellm.types.utils import ModelResponse
+    from litellm.types.utils import ModelResponse, Usage
 
 
 def _completion_response_cost(model_response: "ModelResponse") -> float | None:
@@ -20,6 +21,13 @@ def _completion_response_cost(model_response: "ModelResponse") -> float | None:
     return response_cost if isinstance(response_cost, float) else None
 
 
+def _completion_usage(model_response: "ModelResponse") -> "Usage | None":
+    from litellm.types.utils import Usage
+
+    usage: Final = getattr(model_response, "usage", None)
+    return usage if isinstance(usage, Usage) else None
+
+
 GEMINI_TTS_CHAT_AUDIO_FORMAT: Final = "pcm16"
 GEMINI_TTS_RAW_RESPONSE_FORMAT: Final = "pcm"
 GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS: Final = frozenset({"wav", GEMINI_TTS_RAW_RESPONSE_FORMAT})
@@ -179,4 +187,9 @@ class SpeechToCompletionBridgeTransformationHandler:
         )
         binary_response: Final = HttpxBinaryResponseContent(response)
         binary_response.set_response_cost(_completion_response_cost(model_response))
+        binary_response.usage = _completion_usage(model_response)
+        cached_hidden_params: Final = get_hidden_params(model_response) if served_from_cache(model_response) else None
+        if cached_hidden_params is not None:
+            set_hidden_param(binary_response, "cache_hit", True)
+            set_hidden_param(binary_response, "cache_key", cached_hidden_params.get("cache_key"))
         return binary_response

litellm/experimental_mcp_client/client.py

@@ -498,6 +498,7 @@ class MCPClient:
         elicitation_callback: "ElicitationCallback | None" = None,
         logging_callback: Callable | None = None,
         protocol_version: MCPUpstreamProtocol = "auto",
+        upstream_transport: httpx2.AsyncBaseTransport | None = None,
     ):
         self.protocol_version: MCPUpstreamProtocol = TypeAdapter[MCPUpstreamProtocol](
             MCPUpstreamProtocol
@@ -524,6 +525,7 @@ class MCPClient:
         self._sampling_callback: Callable | None = sampling_callback
         self._elicitation_callback: ElicitationCallback | None = elicitation_callback
         self._logging_callback: Callable | None = logging_callback
+        self._upstream_transport: httpx2.AsyncBaseTransport | None = upstream_transport
         self._session_callbacks_override: _SessionCallbackMap | None = None
         # handle the basic auth value if provided
         if auth_value:
@@ -610,7 +612,11 @@ class MCPClient:
             return stdio_client(server_params), None
         if self.transport_type == MCPTransport.sse:
             headers = self._get_auth_headers()
-            httpx_client_factory = self._create_httpx_client_factory()
+            httpx_client_factory = (
+                self._create_httpx_client_factory()
+                if self._upstream_transport is None
+                else self._create_httpx_client_factory(transport=self._upstream_transport)
+            )
             return (
                 sse_client(
                     url=self.server_url,
@@ -622,7 +628,11 @@ class MCPClient:
             )
         # HTTP transport (default)
         headers = self._get_auth_headers()
-        httpx_client_factory = self._create_httpx_client_factory()
+        httpx_client_factory = (
+            self._create_httpx_client_factory()
+            if self._upstream_transport is None
+            else self._create_httpx_client_factory(transport=self._upstream_transport)
+        )
         verbose_logger.debug("litellm headers for streamable_http_client: %s", headers)
         http_client = httpx_client_factory(
             headers=headers,

litellm/files/main.py

@@ -259,6 +259,7 @@ def create_file(
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
                 create_file_data=_create_file_request,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -388,6 +389,7 @@ def file_retrieve(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -569,6 +571,7 @@ def file_delete(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -775,6 +778,7 @@ def file_list(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -972,6 +976,7 @@ def file_content(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
                 organization=openai_creds.organization,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "azure":
             azure_creds: Final = get_azure_credentials(
@@ -1093,6 +1098,7 @@ def file_content_streaming(
             organization=openai_creds.organization,
             chunk_size=chunk_size,
             client=client if isinstance(client, (OpenAI, AsyncOpenAI)) else None,
+            litellm_params=optional_params.model_dump(exclude_none=True),
         )
     elif custom_llm_provider == LlmProviders.VERTEX_AI.value:
         if not _is_async:

litellm/fine_tuning/main.py

@@ -173,6 +173,7 @@ def create_fine_tuning_job(
     try:
         _is_async: Final = kwargs.pop("acreate_fine_tuning_job", False) is True
         optional_params: Final = GenericLiteLLMParams(**kwargs)
+        provider_model: Final = model.removeprefix(f"{custom_llm_provider}/")
 
         # handle hyperparameters
         hyperparameters = hyperparameters or {}  # original hyperparameters
@@ -218,7 +219,7 @@ def create_fine_tuning_job(
             )
 
             create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
-                model,
+                provider_model,
                 training_file,
                 _oai_hyperparameters,
                 suffix,
@@ -239,6 +240,7 @@ def create_fine_tuning_job(
                 client=kwargs.get(
                     "client", None
                 ),  # note, when we add this to `GenericLiteLLMParams` it impacts a lot of other tests + linting
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":
@@ -262,7 +264,7 @@ def create_fine_tuning_job(
             extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams)
 
             create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
-                model,
+                provider_model,
                 training_file,
                 _oai_hyperparameters,
                 suffix,
@@ -297,7 +299,7 @@ def create_fine_tuning_job(
             response = vertex_fine_tuning_apis_instance.create_fine_tuning_job(
                 _is_async=_is_async,
                 create_fine_tuning_job_data=_build_fine_tuning_job_data(
-                    model,
+                    provider_model,
                     training_file,
                     _oai_hyperparameters,
                     suffix,
@@ -436,6 +438,7 @@ def cancel_fine_tuning_job(
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
                 client=kwargs.get("client", None),
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":
@@ -591,6 +594,7 @@ def list_fine_tuning_jobs(
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
                 client=kwargs.get("client", None),
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":
@@ -730,6 +734,7 @@ def retrieve_fine_tuning_job(
                 max_retries=optional_params.max_retries,
                 _is_async=_is_async,
                 client=kwargs.get("client", None),
+                litellm_params=optional_params.model_dump(exclude_none=True),
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":