Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_key_budgets_endpoint

# Conflicts:
#	tests/test_litellm/proxy/auth/test_auth_exception_handler.py

.github/ci-coverage-allowlist.yml

@@ -49,18 +49,15 @@ test_paths:
     paths:
       - tests/code_coverage_tests/test_aio_http_image_conversion.py
   - reason: >-
-      What is left of a second mirror that sat beside tests/test_litellm and ran nowhere. Its
-      other 30 files moved into the real mirror on 2026-08-20 and now run; these four cannot,
-      because each shares a filename with a live test whose contents are disjoint from it, so
-      landing them means merging test bodies rather than moving a file. Measured on the same
-      date: test_common_utils.py holds 15 tests the live file does not, test_oci_chat_transformation
-      13, test_deepseek_chat_transformation 12, and test_discoverable_endpoints 5. Revisit by
-      merging each into its twin, which is a content review, not a move
+      The last file of a second mirror that sat beside tests/test_litellm and ran nowhere. Its
+      other 33 files landed in the real mirror during August 2026, 30 as moves and 3 by merging
+      their bodies into the live file of the same name. This one cannot follow either route yet:
+      its live twin was rewritten from 1268 lines to 9434, and of the 19 tests here 5 have no
+      counterpart while 25 assertions fail against today's code, so what survives that rewrite
+      is a judgement about the endpoints, not a merge. Revisit by deciding which of the five
+      behaviours still hold
     paths:
-      - tests/litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py
-      - tests/litellm/llms/oci/chat/test_oci_chat_transformation.py
       - tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
-      - tests/litellm/proxy/management_endpoints/test_common_utils.py
   - reason: >-
       No job invokes this suite and its files mix pure transformation tests with ones driving live
       vendor vector stores, so assigning them needs a per-file decision

.github/scripts/assert_ci_coverage.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
 import ast
+import operator
 import pathlib
 import re
 import sys
 import warnings
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Callable, Iterable, Mapping, Sequence
 from dataclasses import dataclass
 from typing import Final
 
@@ -56,6 +57,14 @@ class Allowlist:
         return any(relative_path == path for entry in self.dockerfiles for path in entry.paths)
 
 
+@dataclass(frozen=True, slots=True)
+class Section:
+    name: str
+    entries: tuple[AllowEntry, ...]
+    candidates: tuple[str, ...]
+    matches: Callable[[str, str], bool]
+
+
 @dataclass(frozen=True, slots=True)
 class Scalar:
     key: str
@@ -368,6 +377,25 @@ def _uncovered_dockerfiles(allowlist: Allowlist, tokens: frozenset[str]) -> tupl
     )
 
 
+def _stale_allowlist_paths(
+    allowlist: Allowlist,
+    *,
+    test_files: tuple[str, ...],
+    dockerfiles: tuple[str, ...],
+) -> tuple[Finding, ...]:
+    sections: Final[tuple[Section, ...]] = (
+        Section("test_paths", allowlist.test_paths, test_files, _token_covers),
+        Section("dockerfiles", allowlist.dockerfiles, dockerfiles, operator.eq),
+    )
+    return tuple(
+        Finding(subject=path, detail=f"listed under '{section.name}' but matches no file the census looks at")
+        for section in sections
+        for entry in section.entries
+        for path in entry.paths
+        if not any(section.matches(path, candidate) for candidate in section.candidates)
+    )
+
+
 def _parse_entry(item: object, section: str) -> AllowEntry:
     if not isinstance(item, dict):
         raise SystemExit(f"{ALLOWLIST_FILE.name}: '{section}' entries must be mappings")
@@ -465,7 +493,14 @@ def main() -> int:
 
     test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars))
     dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
+    stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
 
+    if stale_findings:
+        _report(
+            "allowlist entries that exempt nothing",
+            stale_findings,
+            "Delete each from .github/ci-coverage-allowlist.yml; the file it named is gone or was renamed.",
+        )
     if test_findings:
         _report(
             "test files that no CI job invokes",
@@ -478,7 +513,7 @@ def main() -> int:
             dockerfile_findings,
             "Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
         )
-    if test_findings or dockerfile_findings:
+    if stale_findings or test_findings or dockerfile_findings:
         return 1
 
     _write(

.github/scripts/assert_workflow_dir_hygiene.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Three invariants about what lives in .github/workflows/ and what its names mean.
+
+`.github/workflows/` is a directory GitHub reads, not a place to keep things. Every
+file at its top level is parsed as a workflow, so a script or a data file parked there
+is either an invalid workflow or an orphan nobody can find. A subdirectory is not read
+at all, so helper files may live in one. GitHub accepts both `.yml` and `.yaml`, and
+this repo spells them `.yml`, which is a naming rule rather than a validity one and is
+reported separately. And the `_` prefix is the repo's only signal that a workflow is a
+reusable building block rather than something that runs on its own, which is worth
+nothing unless it is true both ways.
+
+    WF001   a top-level file in .github/workflows/ that is not a workflow at all
+    WF002   a workflow whose only trigger is `workflow_call` but is not `_`-prefixed
+    WF003   a `_`-prefixed workflow that no other workflow can call
+    WF004   a real workflow spelled `.yaml` where this directory spells them `.yml`
+
+A workflow with `workflow_call` alongside a human trigger is deliberately dual-mode
+and belongs under its plain name, so only the call-only ones are held to WF002.
+
+Usage
+-----
+    python assert_workflow_dir_hygiene.py
+
+Exit code 1 if any violation is found.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import sys
+from dataclasses import dataclass
+from typing import Final
+
+import yaml
+
+REPO_ROOT: Final = pathlib.Path(__file__).resolve().parents[2]
+WORKFLOW_DIR: Final = REPO_ROOT / ".github" / "workflows"
+SCRIPT_HOME: Final = ".github/scripts/"
+REUSABLE_PREFIX: Final = "_"
+CALL_TRIGGER: Final = "workflow_call"
+CANONICAL_SUFFIX: Final = ".yml"
+WORKFLOW_SUFFIXES: Final = frozenset((CANONICAL_SUFFIX, ".yaml"))
+
+
+@dataclass(frozen=True, slots=True)
+class Finding:
+    subject: str
+    code: str
+    detail: str
+
+    def render(self) -> str:
+        return f"  - {self.subject}: {self.code} {self.detail}"
+
+
+def _triggers(document: object) -> frozenset[str]:
+    if not isinstance(document, dict):
+        return frozenset()
+    raw: Final = document.get("on", document.get(True))
+    if isinstance(raw, str):
+        return frozenset({raw})
+    if isinstance(raw, dict):
+        return frozenset(str(key) for key in raw)
+    if isinstance(raw, list):
+        return frozenset(str(item) for item in raw)
+    return frozenset()
+
+
+def _workflows(directory: pathlib.Path) -> tuple[pathlib.Path, ...]:
+    return tuple(
+        path
+        for path in sorted(directory.iterdir())
+        if path.is_file() and path.suffix in WORKFLOW_SUFFIXES
+    )
+
+
+def _strays(directory: pathlib.Path) -> tuple[Finding, ...]:
+    return tuple(
+        Finding(
+            path.name,
+            "WF001",
+            f"is not a workflow, and GitHub parses every top-level file here as one; "
+            f"move it to {SCRIPT_HOME} or into a subdirectory, which GitHub does not read",
+        )
+        for path in sorted(directory.iterdir())
+        if path.is_file() and path.suffix not in WORKFLOW_SUFFIXES
+    )
+
+
+def _misspelled(directory: pathlib.Path) -> tuple[Finding, ...]:
+    return tuple(
+        Finding(
+            path.name,
+            "WF004",
+            f"is a real workflow and GitHub reads it, but this directory spells them "
+            f"{CANONICAL_SUFFIX}; rename it to {path.stem}{CANONICAL_SUFFIX}",
+        )
+        for path in _workflows(directory)
+        if path.suffix != CANONICAL_SUFFIX
+    )
+
+
+def _misnamed(directory: pathlib.Path) -> tuple[Finding, ...]:
+    return tuple(
+        finding
+        for path in _workflows(directory)
+        for finding in _naming_findings(path, _triggers(yaml.safe_load(path.read_text(encoding="utf-8"))))
+    )
+
+
+def _naming_findings(path: pathlib.Path, triggers: frozenset[str]) -> tuple[Finding, ...]:
+    underscored: Final = path.name.startswith(REUSABLE_PREFIX)
+    if triggers == frozenset({CALL_TRIGGER}) and not underscored:
+        return (
+            Finding(
+                path.name,
+                "WF002",
+                f"is only callable by another workflow, so name it {REUSABLE_PREFIX}{path.name}",
+            ),
+        )
+    if underscored and CALL_TRIGGER not in triggers:
+        return (
+            Finding(
+                path.name,
+                "WF003",
+                f"is named as a reusable workflow but has no {CALL_TRIGGER} trigger; "
+                "add one or drop the prefix",
+            ),
+        )
+    return ()
+
+
+def main() -> int:
+    findings: Final = _strays(WORKFLOW_DIR) + _misspelled(WORKFLOW_DIR) + _misnamed(WORKFLOW_DIR)
+    if not findings:
+        total: Final = len(_workflows(WORKFLOW_DIR))
+        sys.stdout.write(
+            f"OK: {total} workflows, every file in .github/workflows/ is one, and the "
+            f"{REUSABLE_PREFIX} prefix means callable in both directions.\n"
+        )
+        return 0
+    sys.stdout.write("ERROR: .github/workflows/ holds files that break its own conventions\n")
+    for finding in findings:
+        sys.stdout.write(f"{finding.render()}\n")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/_test-unit-base.yml

@@ -129,6 +129,13 @@ jobs:
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
           DIST: ${{ inputs.dist }}
+          # coverage.py's sys.monitoring backend (PEP 669), the cheapest core it has.
+          # It is only the default from Python 3.14, and these shards run 3.12, so it
+          # has to be asked for. Coverage refuses it when branch measurement is on
+          # (`branch_right_left` needs > 3.14.0a5) and falls back to the slow core with
+          # a `no-sysmon` warning, so turning on `branch = true` here means giving this
+          # back until the runners move to 3.14.
+          COVERAGE_CORE: sysmon
         run: |
           if [ "${WORKERS}" = "0" ]; then
             uv run --no-sync pytest ${TEST_PATH:?} \

.github/workflows/ci-coverage.yml

@@ -46,3 +46,6 @@ jobs:
       # nowhere while counting as covered, which is how the caching suite went unrun.
       - name: Assert no -k expression deselects a file from every job that globs it
         run: python .github/scripts/assert_ci_coverage.py --slices
+
+      - name: Assert .github/workflows/ holds only workflows, correctly named
+        run: python .github/scripts/assert_workflow_dir_hygiene.py

.github/workflows/test-linting.yml

@@ -122,6 +122,11 @@ jobs:
           uv run --no-sync ruff check .
           cd ..
 
+      - name: Run Ruff linting (test tree)
+        if: steps.changes.outputs.decision != 'skip'
+        run: |
+          uv run --no-sync ruff check --config ruff-tests.toml tests
+
       - name: Check strict-rule budget (delta vs base)
         if: steps.changes.outputs.decision != 'skip'
         run: |
@@ -132,7 +137,7 @@ jobs:
         run: |
           uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
 
-      - name: Check test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, delta vs base)
+      - name: Check test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs base)
         if: steps.changes.outputs.decision != 'skip'
         run: |
           uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA"

.github/workflows/test-unit.yml

@@ -113,6 +113,7 @@ jobs:
               tests/test_litellm/rag
               tests/test_litellm/realtime_api
               tests/test_litellm/rerank_api
+              tests/test_litellm/rust_bridge
               tests/test_litellm/sandbox
               tests/test_litellm/test_router
               tests/test_litellm/vector_stores

Makefile

@@ -160,6 +160,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 # Linting targets
 lint-ruff: $(LINT_DEP_INSTALL)
 	cd litellm && $(UV_RUN) ruff check . && cd ..
+	$(UV_RUN) ruff check --config ruff-tests.toml tests
 
 # faster linter for developing ...
 # inspiration from:
@@ -205,8 +206,8 @@ lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 	$(UV_RUN) python scripts/type_discipline_gate.py --base origin/litellm_internal_staging
 
 # Test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
-# litellm module-global mutation, credential-gated skips), counted across tests/ the
-# same delta-vs-base way.
+# litellm module-global mutation, credential-gated skips, conftest snapshot
+# inventory), counted across tests/ the same delta-vs-base way.
 lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 	$(UV_RUN) python scripts/test_quality_gate.py --base origin/litellm_internal_staging
 

README.md

@@ -292,6 +292,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 | [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cohere (`cohere`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |
 | [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |

basedpyright-code-budget.json

@@ -105,13 +105,13 @@
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 39017
+    "limit": 39011
   },
   "reportUnknownParameterType": {
     "limit": 19885
   },
   "reportUnknownVariableType": {
-    "limit": 30572
+    "limit": 30569
   },
   "reportUnnecessaryCast": {
     "limit": 117

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -255,6 +255,52 @@ class CheckBatchCost:
             "so it will no longer be polled"
         )
 
+    async def _claim_job_for_costing(self, job: "LiteLLM_ManagedObjectTable") -> bool:
+        """
+        Atomically flip batch_processed from false to true, returning whether this pod won
+        the row. Every pod and uvicorn worker schedules its own poller against the shared
+        table, so without this compare-and-swap two of them can select the same completed
+        batch in one window and both emit an aretrieve_batch spend log for it. Schemas
+        without the column can't be claimed, so they keep the pre-existing behavior.
+
+        Called immediately before the spend log is written rather than before the results
+        fetch, because batch_processed is also what holds off deletion of the files that
+        fetch reads and what keeps an unbilled row selectable by the next poll cycle.
+        """
+        if not self._has_batch_processed_column:
+            return True
+        try:
+            claimed: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+                where={"id": job.id, "batch_processed": False},
+                data={"batch_processed": True},
+            )
+        except Exception as db_err:
+            verbose_proxy_logger.error(
+                f"CheckBatchCost: failed to claim job {job.id} for cost tracking: {db_err}"
+            )
+            return False
+        return claimed > 0
+
+    async def _release_job_claim(self, job: "LiteLLM_ManagedObjectTable") -> None:
+        """Give a claimed row back once billing it failed, so a later poll cycle retries it.
+
+        Safe to match on batch_processed=True: while this poller is active the retrieve
+        path leaves the column alone (batch_cost_poller_is_active), so a true value here
+        is always this pod's own claim.
+        """
+        if not self._has_batch_processed_column:
+            return
+        try:
+            await self.prisma_client.db.litellm_managedobjecttable.update_many(
+                where={"id": job.id, "batch_processed": True},
+                data={"batch_processed": False},
+            )
+        except Exception as db_err:
+            verbose_proxy_logger.error(
+                f"CheckBatchCost: failed to release the claim on job {job.id}, "
+                f"so its cost will not be retried: {db_err}"
+            )
+
     @staticmethod
     def _has_unified_id_without_model(job: "LiteLLM_ManagedObjectTable") -> bool:
         """A unified id that decodes but carries no model_id can never be routed."""
@@ -572,9 +618,10 @@ class CheckBatchCost:
         """
         Fetch a completed batch's results, compute cost/usage, and emit the
         aretrieve_batch spend log. Returns (model_name, llm_provider) on
-        success, None when the job can't be routed to a deployment. Raises on
-        results-fetch or cost-computation failures so the caller can leave the
-        job unprocessed and retry it on a later poll.
+        success, None when the job can't be routed to a deployment or when
+        another pod claimed it. Raises on results-fetch or cost-computation
+        failures so the caller can leave the job unprocessed and retry it on a
+        later poll.
         """
         from litellm.batches.batch_utils import (
             _get_file_content_as_dictionary,
@@ -743,12 +790,23 @@ class CheckBatchCost:
             optional_params={},
         )
 
-        await logging_obj.async_success_handler(
-            result=response,
-            batch_cost=batch_cost,
-            batch_usage=batch_usage,
-            batch_models=batch_models,
-        )
+        if not await self._claim_job_for_costing(job):
+            verbose_proxy_logger.info(
+                f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
+                "in this window, so its cost is already being tracked there"
+            )
+            return None
+
+        try:
+            await logging_obj.async_success_handler(
+                result=response,
+                batch_cost=batch_cost,
+                batch_usage=batch_usage,
+                batch_models=batch_models,
+            )
+        except Exception:
+            await self._release_job_claim(job)
+            raise
 
         # Record batch duration (completed_at - created_at)
         if prom_logger and response.completed_at and response.created_at:

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.57"
+version = "0.1.58"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.57"
+version = "0.1.58"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

helm/litellm-helm/templates/deployment.yaml

@@ -100,6 +100,13 @@ spec:
             - name: DATABASE_URL
               value: {{ .Values.db.url | quote }}
             {{- end }}
+            {{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }}
+            - name: DATABASE_READER_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.secret.name }}
+                  key: {{ .Values.db.secret.readReplicaEndpointKey }}
+            {{- end }}
             {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
             - name: DATABASE_URL_READ_REPLICA
               valueFrom:

helm/litellm-helm/tests/deployment_tests.yaml

@@ -80,6 +80,96 @@ tests:
               secretKeyRef:
                 name: my-secret
                 key: my-key
+  - it: should inject DATABASE_READER_HOST from readReplicaEndpointKey before DATABASE_URL_READ_REPLICA
+    template: deployment.yaml
+    set:
+      db:
+        deployStandalone: false
+        useExisting: true
+        secret:
+          name: postgres
+          usernameKey: username
+          passwordKey: password
+          readReplicaEndpointKey: reader-host
+        readReplicaUrl: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)?sslmode=require
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_READER_HOST
+            valueFrom:
+              secretKeyRef:
+                name: postgres
+                key: reader-host
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_URL_READ_REPLICA
+            value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)?sslmode=require
+      # $(VAR) interpolation only resolves vars defined EARLIER in the env
+      # array, so the reader host must precede the composed URL
+      - equal:
+          path: spec.template.spec.containers[0].env[7].name
+          value: DATABASE_READER_HOST
+      - equal:
+          path: spec.template.spec.containers[0].env[8].name
+          value: DATABASE_URL_READ_REPLICA
+  - it: should omit reader host when readReplicaUrl is unset
+    template: deployment.yaml
+    set:
+      db:
+        deployStandalone: false
+        useExisting: true
+        secret:
+          name: postgres
+          usernameKey: username
+          passwordKey: password
+          readReplicaEndpointKey: reader-host
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_READER_HOST
+            valueFrom:
+              secretKeyRef:
+                name: postgres
+                key: reader-host
+  - it: should prefer readReplicaUrlKey over readReplicaEndpointKey composition
+    template: deployment.yaml
+    set:
+      db:
+        useExisting: true
+        secret:
+          name: postgres
+          usernameKey: username
+          passwordKey: password
+          readReplicaUrlKey: reader-url
+          readReplicaEndpointKey: reader-host
+        readReplicaUrl: postgresql://ignored
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_URL_READ_REPLICA
+            valueFrom:
+              secretKeyRef:
+                name: postgres
+                key: reader-url
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_URL_READ_REPLICA
+            value: postgresql://ignored
+      # the unused reader-host secret ref must be suppressed so a missing
+      # key can't fail pod creation
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_READER_HOST
+            valueFrom:
+              secretKeyRef:
+                name: postgres
+                key: reader-host
   - it: should work with extraEnvVars
     template: deployment.yaml
     set:

helm/litellm-helm/values.yaml

@@ -277,6 +277,14 @@ db:
     # written to db.readReplicaUrl ends up visible in the rendered pod spec
     # and the Helm release secret.
     readReplicaUrlKey: ""
+    # Optional: when set, a DATABASE_READER_HOST env var is sourced from this
+    # secret key, so db.readReplicaUrl can compose the reader URL from
+    # individual secret components, e.g.
+    # postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)
+    # Use this when your secret store holds the bare reader hostname rather
+    # than a full connection URL. Only takes effect when readReplicaUrl is
+    # set; ignored when readReplicaUrlKey is set.
+    readReplicaEndpointKey: ""
 
   # Optional read-replica routing. When set, the proxy sends read-only
   # queries (find_*, count, group_by, query_raw/_first) to this URL while

helm/litellm/templates/_helpers.tpl

@@ -213,18 +213,21 @@ whenever the password contains a URL-reserved character (@, /, ?, %, +,
 
 When `database.writer.useIAMAuth: true`, the chart injects
 IAM_TOKEN_DB_AUTH=true and omits DATABASE_PASSWORD — the entrypoint mints
-the URL from DATABASE_HOST/PORT/USER/NAME plus a short-lived IAM token
-instead of a static password.
+the URL from DATABASE_HOST/PORT/USER/NAME plus a short-lived AWS RDS IAM
+token instead of a static password. `database.writer.useAzureEntraAuth: true`
+does the same with AZURE_POSTGRESQL_AUTH=true and a Microsoft Entra ID token,
+for Azure Database for PostgreSQL. The two are mutually exclusive.
 
 The read replica is opt-in via `database.reader.host`. The chart emits
 DATABASE_HOST_READ_REPLICA / DATABASE_PORT_READ_REPLICA /
 DATABASE_NAME_READ_REPLICA (+ DATABASE_SCHEMA_READ_REPLICA) for both auth
 modes, plus DATABASE_USER_READ_REPLICA / DATABASE_PASSWORD_READ_REPLICA for
-password auth. When `database.reader.useIAMAuth: true` it omits
+password auth. When `database.reader.useIAMAuth: true` (or
+`database.reader.useAzureEntraAuth: true`) it omits
 DATABASE_PASSWORD_READ_REPLICA and the entrypoint mints the reader URL the
-same way. Reader IAM only takes effect when the writer also uses IAM auth
-(the proxy gates URL minting on IAM_TOKEN_DB_AUTH, which only the writer
-sets).
+same way. Reader token auth only takes effect when the writer uses the same
+token source, since the proxy gates URL minting on the single global
+IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets.
 */}}
 {{- define "litellm.serverEnv" -}}
 {{- $root := .root -}}
@@ -254,9 +257,15 @@ sets).
 - name: DATABASE_SCHEMA
   value: {{ .schema | quote }}
 {{- end }}
+{{- if and .useIAMAuth .useAzureEntraAuth }}
+{{- fail "database.writer.useIAMAuth and database.writer.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source" }}
+{{- end }}
 {{- if .useIAMAuth }}
 - name: IAM_TOKEN_DB_AUTH
   value: "true"
+{{- else if .useAzureEntraAuth }}
+- name: AZURE_POSTGRESQL_AUTH
+  value: "true"
 {{- else }}
 - name: DATABASE_PASSWORD
   valueFrom:
@@ -270,6 +279,9 @@ sets).
 {{- if and .useIAMAuth (not $root.Values.database.writer.useIAMAuth) }}
 {{- fail "database.reader.useIAMAuth requires database.writer.useIAMAuth: true (the proxy gates IAM URL minting on IAM_TOKEN_DB_AUTH, which is only set by the writer)" }}
 {{- end }}
+{{- if and .useAzureEntraAuth (not $root.Values.database.writer.useAzureEntraAuth) }}
+{{- fail "database.reader.useAzureEntraAuth requires database.writer.useAzureEntraAuth: true (the proxy gates Entra URL minting on AZURE_POSTGRESQL_AUTH, which is only set by the writer)" }}
+{{- end }}
 - name: DATABASE_HOST_READ_REPLICA
   value: {{ .host | quote }}
 - name: DATABASE_PORT_READ_REPLICA
@@ -280,7 +292,7 @@ sets).
 - name: DATABASE_SCHEMA_READ_REPLICA
   value: {{ .schema | quote }}
 {{- end }}
-{{- if .useIAMAuth }}
+{{- if or .useIAMAuth .useAzureEntraAuth }}
 {{- if .passwordSecret.name }}
 - name: DATABASE_USER_READ_REPLICA
   valueFrom:

helm/litellm/tests/database_auth_tests.yaml

@@ -0,0 +1,116 @@
+suite: test database token auth env vars
+templates:
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - backend/configmap.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: writer emits DATABASE_PASSWORD and no token toggle by default
+    template: gateway/deployment.yaml
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: litellm-writer-secret
+                key: password
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: IAM_TOKEN_DB_AUTH
+            value: "true"
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: AZURE_POSTGRESQL_AUTH
+            value: "true"
+          any: true
+
+  - it: writer emits AZURE_POSTGRESQL_AUTH and omits DATABASE_PASSWORD under Entra auth
+    template: gateway/deployment.yaml
+    set:
+      database.writer.useAzureEntraAuth: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: AZURE_POSTGRESQL_AUTH
+            value: "true"
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_PASSWORD
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: IAM_TOKEN_DB_AUTH
+            value: "true"
+          any: true
+
+  - it: backend gets the same Entra toggle as the gateway
+    template: backend/deployment.yaml
+    set:
+      database.writer.useAzureEntraAuth: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: AZURE_POSTGRESQL_AUTH
+            value: "true"
+          any: true
+
+  - it: writer rejects both token sources at once
+    template: gateway/deployment.yaml
+    set:
+      database.writer.useIAMAuth: true
+      database.writer.useAzureEntraAuth: true
+    asserts:
+      - failedTemplate:
+          errorMessage: "database.writer.useIAMAuth and database.writer.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source"
+
+  - it: reader Entra auth without writer Entra auth is rejected
+    template: gateway/deployment.yaml
+    set:
+      database.reader.host: reader.example.com
+      database.reader.dbname: litellm
+      database.reader.useAzureEntraAuth: true
+    asserts:
+      - failedTemplate:
+          errorMessage: "database.reader.useAzureEntraAuth requires database.writer.useAzureEntraAuth: true (the proxy gates Entra URL minting on AZURE_POSTGRESQL_AUTH, which is only set by the writer)"
+
+  - it: reader under Entra auth omits DATABASE_PASSWORD_READ_REPLICA
+    template: gateway/deployment.yaml
+    set:
+      database.writer.useAzureEntraAuth: true
+      database.reader.host: reader.example.com
+      database.reader.dbname: litellm
+      database.reader.useAzureEntraAuth: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_HOST_READ_REPLICA
+            value: reader.example.com
+          any: true
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_USER_READ_REPLICA
+            valueFrom:
+              secretKeyRef:
+                name: litellm-reader-secret
+                key: username
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_PASSWORD_READ_REPLICA
+          any: true

helm/litellm/values.yaml

@@ -145,6 +145,8 @@ database:
     dbname: ""
     schema: ""
     useIAMAuth: false
+    # Azure Database for PostgreSQL with a Microsoft Entra ID token; mutually exclusive with useIAMAuth
+    useAzureEntraAuth: false
     passwordSecret:
       name: litellm-writer-secret
       usernameKey: username
@@ -159,6 +161,8 @@ database:
     dbname: ""
     schema: ""
     useIAMAuth: false
+    # Azure Database for PostgreSQL with a Microsoft Entra ID token; mutually exclusive with useIAMAuth
+    useAzureEntraAuth: false
     passwordSecret:
       name: litellm-reader-secret
       usernameKey: username

litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_shadow_eval_max_budget/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN     "max_budget" DOUBLE PRECISION;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN     "shadow_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1502,7 +1502,8 @@ model LiteLLM_ShadowEvalJob {
   baseline_model    String?  // reverse only: the fixed model the router is judged against
   judge_model       String
   shadow_percentage Float
-  max_turns         Int      // this key's sample budget: judge at most this many turns
+  max_turns         Int      // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
+  max_budget        Float?   // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
   created_at        DateTime @default(now())
   created_by        String?
   ends_at           DateTime
@@ -1525,6 +1526,7 @@ model LiteLLM_ShadowEvalAttempt {
   shadow_model String?
   confidence   Float?
   judge_cost   Float    @default(0)
+  shadow_cost  Float    @default(0)
   error        String?
   created_at   DateTime @default(now())
 

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.87"
+version = "0.4.88"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.87"
+version = "0.4.88"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs

@@ -295,6 +295,8 @@ fn core_error_kind(error: &CoreError) -> &'static str {
         CoreError::Http { .. } => "HttpError",
         CoreError::InvalidResponse(_) => "InvalidResponse",
         CoreError::Network(_) => "NetworkError",
+        CoreError::Connect(_) => "ConnectError",
         CoreError::Routing(_) => "RoutingError",
+        CoreError::Unsupported(_) => "UnsupportedRequest",
     }
 }

litellm-rust/crates/ai-gateway/src/ocr/hooks.rs

@@ -324,6 +324,8 @@ fn core_error_kind(error: &CoreError) -> &'static str {
         CoreError::Http { .. } => "HttpError",
         CoreError::InvalidResponse(_) => "InvalidResponse",
         CoreError::Network(_) => "NetworkError",
+        CoreError::Connect(_) => "ConnectError",
         CoreError::Routing(_) => "RoutingError",
+        CoreError::Unsupported(_) => "UnsupportedRequest",
     }
 }

litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs

@@ -105,12 +105,20 @@ impl IntoResponse for MessagesRouteError {
             ),
             CoreError::Http { .. }
             | CoreError::Network(_)
+            | CoreError::Connect(_)
             | CoreError::InvalidResponse(_)
             | CoreError::InvalidType { .. }
             | CoreError::MissingField(_) => (
                 StatusCode::BAD_GATEWAY,
                 "messages provider request failed".to_string(),
             ),
+            // The gateway has no Python implementation to decline to, so a
+            // request the core cannot serve is reported to the caller. The
+            // reason is a fixed internal string, never provider content.
+            CoreError::Unsupported(reason) => (
+                StatusCode::BAD_REQUEST,
+                format!("messages request is not supported: {reason}"),
+            ),
         };
         (
             status,

litellm-rust/crates/core/src/chat_completions/client.rs

@@ -0,0 +1,15 @@
+use std::sync::OnceLock;
+use std::time::Duration;
+
+use crate::constants::{CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS, CHAT_COMPLETIONS_TIMEOUT_SECS};
+
+pub(super) fn http_client() -> &'static reqwest::Client {
+    static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
+    CLIENT.get_or_init(|| {
+        reqwest::Client::builder()
+            .timeout(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS))
+            .connect_timeout(Duration::from_secs(CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS))
+            .build()
+            .unwrap_or_else(|_| reqwest::Client::new())
+    })
+}

litellm-rust/crates/core/src/chat_completions/common_utils.rs

@@ -0,0 +1,28 @@
+use serde_json::{Map, Value};
+
+use crate::error::CoreResult;
+use crate::http_utils::string_headers as shared_string_headers;
+use crate::providers::anthropic::chat_completions::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG;
+
+use super::transformation::ChatCompletionsProviderConfig;
+
+const HEADER_CONTEXT: &str = "chat completions";
+
+pub(super) fn chat_completions_provider_config(
+    provider: &str,
+) -> Option<&'static dyn ChatCompletionsProviderConfig> {
+    match provider {
+        "anthropic" => Some(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG),
+        #[cfg(feature = "bedrock-auth")]
+        "bedrock" => Some(
+            &crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
+        ),
+        _ => None,
+    }
+}
+
+pub(super) fn string_headers(
+    extra_headers: Option<Map<String, Value>>,
+) -> CoreResult<Vec<(String, String)>> {
+    shared_string_headers(HEADER_CONTEXT, extra_headers)
+}

litellm-rust/crates/core/src/chat_completions/conversation.rs

@@ -0,0 +1,254 @@
+//! Provider-neutral conversation shape.
+//!
+//! Both Anthropic Messages and Bedrock Converse want the same thing out of an
+//! OpenAI message list: the system prompt lifted out, consecutive same-role
+//! turns merged, and text blocks that are never empty. That normalization is
+//! shared here so a provider config only renders the result into its own wire
+//! shape.
+//!
+//! Mirrors Python's `anthropic_messages_pt` /
+//! `_bedrock_converse_messages_pt` for the text-only surface this route
+//! accepts; anything richer is declined upstream by the capability gate.
+
+use crate::constants::EMPTY_TEXT_PLACEHOLDER;
+
+use super::types::{ChatMessage, ChatMessageContent};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum TurnRole {
+    User,
+    Assistant,
+}
+
+impl TurnRole {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::User => "user",
+            Self::Assistant => "assistant",
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Turn {
+    pub role: TurnRole,
+    pub texts: Vec<String>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Conversation {
+    pub system: Vec<String>,
+    pub turns: Vec<Turn>,
+}
+
+/// True when the conversation can be sent as-is.
+///
+/// Python inserts a placeholder first user turn only under
+/// `litellm.modify_params`, which the core cannot see, so a conversation that
+/// does not open on a user turn is declined rather than guessed at.
+impl Conversation {
+    pub fn opens_on_user_turn(&self) -> bool {
+        self.turns
+            .first()
+            .is_some_and(|turn| turn.role == TurnRole::User)
+    }
+}
+
+fn message_texts(content: &ChatMessageContent) -> Vec<String> {
+    match content {
+        ChatMessageContent::Text(text) => vec![text.clone()],
+        ChatMessageContent::Parts(parts) => parts
+            .iter()
+            .filter_map(|part| part.get("text").and_then(|text| text.as_str()))
+            .map(str::to_string)
+            .collect(),
+    }
+}
+
+/// Python rewrites empty or whitespace-only text rather than dropping it, so an
+/// entirely empty content list never reaches a provider that rejects one.
+fn sanitize(text: String) -> String {
+    if text.trim().is_empty() {
+        return EMPTY_TEXT_PLACEHOLDER.to_string();
+    }
+    text
+}
+
+pub fn build_conversation(messages: &[ChatMessage]) -> Conversation {
+    let system = messages
+        .iter()
+        .filter(|message| message.role == "system")
+        .filter_map(|message| message.content.as_ref())
+        .flat_map(message_texts)
+        .filter(|text| !text.is_empty())
+        .collect();
+
+    let turns = messages
+        .iter()
+        .filter(|message| message.role != "system")
+        .fold(Vec::<Turn>::new(), |mut turns, message| {
+            let role = if message.role == "assistant" {
+                TurnRole::Assistant
+            } else {
+                TurnRole::User
+            };
+            let texts = message
+                .content
+                .as_ref()
+                .map(message_texts)
+                .unwrap_or_default()
+                .into_iter()
+                .map(sanitize);
+            match turns.last_mut() {
+                Some(last) if last.role == role => last.texts.extend(texts),
+                _ => turns.push(Turn {
+                    role,
+                    texts: texts.collect(),
+                }),
+            }
+            turns
+        });
+
+    // Anthropic and Bedrock both reject trailing whitespace on the final
+    // assistant turn, so Python right-strips it there; mirror that exactly.
+    let turns = match turns.split_last() {
+        Some((last, rest)) if last.role == TurnRole::Assistant => rest
+            .iter()
+            .cloned()
+            .chain([Turn {
+                role: last.role,
+                texts: last
+                    .texts
+                    .iter()
+                    .map(|text| text.trim_end().to_string())
+                    .collect(),
+            }])
+            .collect(),
+        _ => turns,
+    };
+
+    Conversation { system, turns }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    fn messages(value: serde_json::Value) -> Vec<ChatMessage> {
+        serde_json::from_value(value).expect("valid messages")
+    }
+
+    #[test]
+    fn lifts_system_messages_out_of_the_turn_list() {
+        let conversation = build_conversation(&messages(json!([
+            {"role": "system", "content": "be terse"},
+            {"role": "user", "content": "hi"}
+        ])));
+        assert_eq!(conversation.system, vec!["be terse".to_string()]);
+        assert_eq!(
+            conversation.turns,
+            vec![Turn {
+                role: TurnRole::User,
+                texts: vec!["hi".to_string()]
+            }]
+        );
+    }
+
+    #[test]
+    fn merges_consecutive_same_role_turns() {
+        let conversation = build_conversation(&messages(json!([
+            {"role": "user", "content": "one"},
+            {"role": "user", "content": "two"},
+            {"role": "assistant", "content": "ack"},
+            {"role": "user", "content": "three"}
+        ])));
+        assert_eq!(
+            conversation.turns,
+            vec![
+                Turn {
+                    role: TurnRole::User,
+                    texts: vec!["one".to_string(), "two".to_string()]
+                },
+                Turn {
+                    role: TurnRole::Assistant,
+                    texts: vec!["ack".to_string()]
+                },
+                Turn {
+                    role: TurnRole::User,
+                    texts: vec!["three".to_string()]
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn flattens_text_parts_in_order() {
+        let conversation = build_conversation(&messages(json!([
+            {"role": "user", "content": [
+                {"type": "text", "text": "first"},
+                {"type": "text", "text": "second"}
+            ]}
+        ])));
+        assert_eq!(
+            conversation.turns[0].texts,
+            vec!["first".to_string(), "second".to_string()]
+        );
+    }
+
+    #[test]
+    fn rewrites_empty_and_whitespace_only_text_to_the_python_placeholder() {
+        let conversation = build_conversation(&messages(json!([
+            {"role": "user", "content": ""},
+            {"role": "assistant", "content": "   "},
+            {"role": "user", "content": "real"}
+        ])));
+        assert_eq!(conversation.turns[0].texts, vec![EMPTY_TEXT_PLACEHOLDER]);
+        assert_eq!(conversation.turns[1].texts, vec![EMPTY_TEXT_PLACEHOLDER]);
+    }
+
+    #[test]
+    fn right_strips_only_the_final_assistant_turn() {
+        let conversation = build_conversation(&messages(json!([
+            {"role": "user", "content": "hi"},
+            {"role": "assistant", "content": "kept  "},
+            {"role": "user", "content": "more"},
+            {"role": "assistant", "content": "stripped  "}
+        ])));
+        assert_eq!(conversation.turns[1].texts, vec!["kept  ".to_string()]);
+        assert_eq!(conversation.turns[3].texts, vec!["stripped".to_string()]);
+    }
+
+    #[test]
+    fn does_not_strip_when_the_last_turn_is_a_user_turn() {
+        let conversation = build_conversation(&messages(json!([
+            {"role": "assistant", "content": "kept  "},
+            {"role": "user", "content": "hi  "}
+        ])));
+        assert_eq!(conversation.turns[0].texts, vec!["kept  ".to_string()]);
+        assert_eq!(conversation.turns[1].texts, vec!["hi  ".to_string()]);
+    }
+
+    #[test]
+    fn reports_whether_the_conversation_opens_on_a_user_turn() {
+        assert!(
+            build_conversation(&messages(json!([{"role": "user", "content": "hi"}])))
+                .opens_on_user_turn()
+        );
+        assert!(
+            !build_conversation(&messages(json!([{"role": "assistant", "content": "hi"}])))
+                .opens_on_user_turn()
+        );
+        assert!(!Conversation::default().opens_on_user_turn());
+    }
+
+    #[test]
+    fn drops_empty_system_text_the_way_python_skips_empty_system_blocks() {
+        let conversation = build_conversation(&messages(json!([
+            {"role": "system", "content": ""},
+            {"role": "system", "content": "kept"},
+            {"role": "user", "content": "hi"}
+        ])));
+        assert_eq!(conversation.system, vec!["kept".to_string()]);
+    }
+}

litellm-rust/crates/core/src/chat_completions/handler.rs

@@ -0,0 +1,147 @@
+use serde_json::Value;
+
+use crate::error::{CoreError, CoreResult};
+use crate::http_utils::truncate_error_body;
+
+use super::client::http_client;
+use super::transformation::ChatCompletionsAuth;
+use super::types::{
+    ChatCompletionsResponse, ProviderChatCompletionsRequest, ProviderChatResponseData,
+};
+
+pub(super) async fn execute_chat_completions_provider_call(
+    request: ProviderChatCompletionsRequest,
+) -> CoreResult<ChatCompletionsResponse> {
+    let body = serde_json::to_vec(&request.body).map_err(|err| {
+        CoreError::InvalidRequest(format!(
+            "failed to serialize chat completions request: {err}"
+        ))
+    })?;
+    let headers = signed_headers(&request, &body).await?;
+
+    let mut request_builder = http_client().post(&request.url).body(body);
+    for (key, value) in &headers {
+        request_builder = request_builder.header(key, value);
+    }
+    if let Some(duration) = request.timeout {
+        request_builder = request_builder.timeout(duration);
+    }
+
+    let response = request_builder.send().await.map_err(|err| {
+        // Failing to establish the connection means the request never went out,
+        // so the host can still serve it. Everything else here, a timeout
+        // above all, may have reached the provider and been answered.
+        if err.is_connect() || err.is_builder() {
+            CoreError::Connect(err.to_string())
+        } else {
+            CoreError::Network(err.to_string())
+        }
+    })?;
+
+    let status = response.status();
+    let text = response
+        .text()
+        .await
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+
+    if !status.is_success() {
+        return Err(CoreError::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&text),
+        });
+    }
+
+    let body: Value = serde_json::from_str(&text).map_err(|err| {
+        CoreError::InvalidResponse(format!("invalid chat completions response JSON: {err}"))
+    })?;
+    request
+        .config
+        .transform_response(&request.model, ProviderChatResponseData { body })
+        .map_err(as_response_error)
+}
+
+/// Re-tag an error raised while normalizing a response the provider already
+/// returned.
+///
+/// A config reports the same variants on either side of the call: a missing
+/// field or an unsupported block can mean "this request cannot be translated"
+/// during prepare and "this response cannot be normalized" here. Only the
+/// second kind has already been billed, and a host that keeps a reference
+/// implementation must not retry those, so collapse them to one variant that
+/// can only mean the provider was already called.
+pub(super) fn as_response_error(err: CoreError) -> CoreError {
+    match err {
+        already @ (CoreError::InvalidResponse(_) | CoreError::Http { .. }) => already,
+        other => CoreError::InvalidResponse(other.to_string()),
+    }
+}
+
+#[cfg(feature = "bedrock-auth")]
+pub(super) async fn signed_headers(
+    request: &ProviderChatCompletionsRequest,
+    body: &[u8],
+) -> CoreResult<Vec<(String, String)>> {
+    use std::collections::BTreeMap;
+    use std::time::SystemTime;
+
+    use crate::providers::bedrock::aws_base::{
+        aws_auth_config, aws_signature_headers, host_supplied_credentials,
+        is_sigv4_computed_header, resolve_credentials, sign_bedrock_post,
+    };
+
+    let ChatCompletionsAuth::AwsSigV4 { region } = &request.auth else {
+        return Ok(request.upstream_headers.clone());
+    };
+    // Reattaching a header the signer also emits would put both copies on the
+    // wire, and Bedrock rejects that pair. Python instead drops the caller's
+    // copy and prefers a forwarded Authorization over the signature, so leave
+    // the request to Python rather than serving it a different way here.
+    if request
+        .upstream_headers
+        .iter()
+        .any(|(name, _)| is_sigv4_computed_header(name))
+    {
+        return Err(CoreError::Unsupported(
+            "request forwards a header AWS SigV4 computes",
+        ));
+    }
+    let env_lookup = |key: &str| std::env::var(key).ok();
+    let unsigned: BTreeMap<String, String> = request.upstream_headers.iter().cloned().collect();
+    // A host with its own resolution chain hands the result down; only fall
+    // back to deriving credentials here when it supplied none.
+    let credentials = match host_supplied_credentials(&request.optional_params) {
+        Some(credentials) => credentials,
+        None => {
+            resolve_credentials(
+                aws_auth_config(&request.optional_params, &env_lookup),
+                &env_lookup,
+            )
+            .await?
+        }
+    };
+    let signature = sign_bedrock_post(
+        &request.url,
+        body,
+        &aws_signature_headers(&unsigned),
+        region,
+        &credentials,
+        SystemTime::now(),
+    )?;
+    // Every original header goes back on the wire alongside the computed ones,
+    // as Python reattaches them. The guard above already rejected the names
+    // that would collide, so no name appears twice.
+    Ok(unsigned.into_iter().chain(signature).collect())
+}
+
+#[cfg(not(feature = "bedrock-auth"))]
+pub(super) async fn signed_headers(
+    request: &ProviderChatCompletionsRequest,
+    _body: &[u8],
+) -> CoreResult<Vec<(String, String)>> {
+    match &request.auth {
+        ChatCompletionsAuth::AwsSigV4 { .. } => Err(CoreError::Unsupported(
+            "AWS SigV4 requires the bedrock-auth feature",
+        )),
+        _ => Ok(request.upstream_headers.clone()),
+    }
+}

litellm-rust/crates/core/src/chat_completions/mod.rs

@@ -0,0 +1,59 @@
+//! The `/chat/completions` call, the Rust equivalent of Python's
+//! `litellm.completion()`.
+//!
+//! [`chat_completions`] is the top-level entrypoint: give it a model, the
+//! OpenAI-shaped message list, the provider-mapped optional params, and
+//! credentials, and it resolves the provider, translates the conversation,
+//! calls the provider, and returns a typed OpenAI-shaped response.
+
+mod client;
+mod common_utils;
+pub mod conversation;
+pub(crate) mod handler;
+mod prepare;
+pub mod response_utils;
+pub mod transformation;
+pub mod types;
+
+use serde_json::{Map, Value};
+
+use crate::error::CoreResult;
+
+use handler::execute_chat_completions_provider_call;
+use prepare::{parse_messages, prepare_chat_completions_call, resolve_provider_config};
+use types::{ChatCompletionsRequest, ChatCompletionsResponse};
+
+pub async fn chat_completions(
+    request: ChatCompletionsRequest<'_>,
+) -> CoreResult<ChatCompletionsResponse> {
+    execute_chat_completions_provider_call(prepare_chat_completions_call(request)?).await
+}
+
+/// Whether the core would accept this request, without resolving credentials or
+/// touching the network.
+///
+/// A host that keeps the Python implementation asks this first so it can emit
+/// its pre-call logging exactly once, on whichever path is about to run.
+/// Returns the decline reason, or `None` when the request is accepted.
+pub fn chat_completions_decline_reason(
+    model: &str,
+    custom_llm_provider: Option<&str>,
+    messages: Value,
+    optional_params: &Map<String, Value>,
+) -> Option<&'static str> {
+    let Ok((_, config)) = resolve_provider_config(model, custom_llm_provider) else {
+        return Some("provider is not on the rust chat completions path");
+    };
+    let Ok(messages) = parse_messages(messages) else {
+        return Some("unreadable message list");
+    };
+    if messages.is_empty() {
+        return Some("empty message list");
+    }
+    config
+        .unsupported_reason(&messages, optional_params)
+        .map(|reason| reason.0)
+}
+
+#[cfg(test)]
+mod tests;

litellm-rust/crates/core/src/chat_completions/prepare.rs

@@ -0,0 +1,118 @@
+use serde_json::Value;
+
+use crate::error::{CoreError, CoreResult};
+use crate::http_utils::has_header;
+use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
+
+use super::common_utils::{chat_completions_provider_config, string_headers};
+use super::transformation::{ChatCompletionsAuth, ChatCompletionsProviderConfig};
+use super::types::{ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsRequest};
+
+pub(super) fn resolve_provider_config<'a>(
+    model: &'a str,
+    custom_llm_provider: Option<&'a str>,
+) -> CoreResult<(String, &'static dyn ChatCompletionsProviderConfig)> {
+    let provider_info = get_custom_llm_provider(model, custom_llm_provider)
+        .or_else(|| {
+            custom_llm_provider.map(|provider| CustomLlmProvider {
+                model,
+                custom_llm_provider: provider,
+            })
+        })
+        .ok_or_else(|| {
+            CoreError::InvalidProvider(
+                "unable to resolve custom_llm_provider for chat completions request".to_string(),
+            )
+        })?;
+    let config = chat_completions_provider_config(provider_info.custom_llm_provider)
+        .ok_or_else(|| CoreError::InvalidProvider(provider_info.custom_llm_provider.to_string()))?;
+    Ok((provider_info.model.to_string(), config))
+}
+
+pub(super) fn parse_messages(messages: Value) -> CoreResult<Vec<ChatMessage>> {
+    serde_json::from_value(messages).map_err(|err| {
+        CoreError::InvalidRequest(format!("invalid chat completions messages: {err}"))
+    })
+}
+
+pub(super) fn prepare_chat_completions_call(
+    request: ChatCompletionsRequest<'_>,
+) -> CoreResult<ProviderChatCompletionsRequest> {
+    let (model, config) = resolve_provider_config(request.model, request.custom_llm_provider)?;
+    let env_lookup = |key: &str| std::env::var(key).ok();
+
+    let messages = parse_messages(request.messages)?;
+    if messages.is_empty() {
+        return Err(CoreError::InvalidRequest(
+            "chat completions requires at least one message".to_string(),
+        ));
+    }
+    if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) {
+        return Err(CoreError::Unsupported(reason.0));
+    }
+
+    let mut headers = string_headers(request.extra_headers)?;
+    let auth = config.auth(
+        request.api_key,
+        &model,
+        &request.optional_params,
+        &env_lookup,
+    )?;
+    match &auth {
+        ChatCompletionsAuth::Header { name, value } => {
+            // The deployment's credential replaces whatever the caller forwarded
+            // under the same name, mirroring Python's
+            // `{**headers, **anthropic_headers}`: letting a request header win
+            // would let its sender choose the principal the call bills to.
+            //
+            // The exception is a scheme the provider hands off to entirely, such
+            // as an Anthropic OAuth bearer, where Python drops `x-api-key`
+            // instead of resolving one. Re-adding it there would put the
+            // credential into a header the host removed on purpose.
+            if !config.defers_to_forwarded_auth(&headers) {
+                headers.retain(|(header, _)| !header.eq_ignore_ascii_case(name));
+                headers.push(((*name).to_string(), value.clone()));
+            }
+        }
+        ChatCompletionsAuth::Bearer { token } => {
+            // Bedrock's `get_request_headers` assigns `headers["Authorization"]`
+            // unconditionally once a bearer token resolves, so the deployment's
+            // identity outranks whatever the caller forwarded. Keeping the
+            // caller's would bill and authorize the call as a different
+            // principal than the same deployment uses on Python.
+            //
+            // The `Header` arm below keeps the opposite precedence on purpose:
+            // Anthropic's transform honours a forwarded OAuth bearer.
+            headers.retain(|(name, _)| !name.eq_ignore_ascii_case("authorization"));
+            headers.push(("authorization".to_string(), format!("Bearer {token}")));
+        }
+        // SigV4 signs the serialized body, so the handler adds its headers.
+        ChatCompletionsAuth::AwsSigV4 { .. } => {}
+    }
+
+    for (name, value) in config.default_headers() {
+        if !has_header(&headers, name) {
+            headers.push(((*name).to_string(), (*value).to_string()));
+        }
+    }
+
+    let url = config.complete_url(
+        request.api_base,
+        &model,
+        &request.optional_params,
+        &env_lookup,
+    )?;
+    let transformed =
+        config.transform_request(&model, messages, request.optional_params.clone())?;
+
+    Ok(ProviderChatCompletionsRequest {
+        model,
+        config,
+        url,
+        body: transformed.body,
+        upstream_headers: headers,
+        auth,
+        optional_params: request.optional_params,
+        timeout: request.timeout,
+    })
+}

litellm-rust/crates/core/src/chat_completions/response_utils.rs

@@ -0,0 +1,101 @@
+//! Response normalization shared by every chat completions provider config.
+
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use super::types::{ChatCompletionsUsage, PromptTokensDetails};
+
+/// OpenAI finish reasons, mirroring Python's `_FINISH_REASON_MAP` for the
+/// reasons the providers on this route can emit. Python warns and falls back to
+/// `stop` for anything unmapped, so do the same.
+const FINISH_REASONS: &[(&str, &str)] = &[
+    ("end_turn", "stop"),
+    ("stop_sequence", "stop"),
+    ("max_tokens", "length"),
+    ("refusal", "content_filter"),
+    ("compaction", "length"),
+    ("guardrail_intervened", "content_filter"),
+    ("content_filtered", "content_filter"),
+    ("content_filter", "content_filter"),
+    ("stop", "stop"),
+    ("length", "length"),
+];
+
+pub fn finish_reason_for(provider_reason: &str) -> &'static str {
+    FINISH_REASONS
+        .iter()
+        .find(|(reason, _)| *reason == provider_reason)
+        .map_or("stop", |(_, mapped)| *mapped)
+}
+
+/// Python folds cache tokens into `prompt_tokens` and reports the split under
+/// `prompt_tokens_details`; mirror that so cost tracking agrees on both paths.
+pub fn usage_from_parts(
+    input_tokens: u64,
+    output_tokens: u64,
+    cache_read_tokens: u64,
+    cache_creation_tokens: u64,
+) -> ChatCompletionsUsage {
+    let prompt_tokens = input_tokens + cache_read_tokens + cache_creation_tokens;
+    ChatCompletionsUsage {
+        prompt_tokens,
+        completion_tokens: output_tokens,
+        total_tokens: prompt_tokens + output_tokens,
+        prompt_tokens_details: PromptTokensDetails {
+            cached_tokens: cache_read_tokens,
+            cache_creation_tokens,
+            text_tokens: input_tokens,
+        },
+    }
+}
+
+pub fn unix_now() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_or(0, |elapsed| elapsed.as_secs())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn maps_every_reason_the_route_can_observe() {
+        assert_eq!(finish_reason_for("end_turn"), "stop");
+        assert_eq!(finish_reason_for("stop_sequence"), "stop");
+        assert_eq!(finish_reason_for("max_tokens"), "length");
+        assert_eq!(finish_reason_for("refusal"), "content_filter");
+        assert_eq!(finish_reason_for("guardrail_intervened"), "content_filter");
+        // Converse emits these two, and folding them into `stop` would report a
+        // filtered completion as a normal one.
+        assert_eq!(finish_reason_for("content_filtered"), "content_filter");
+        assert_eq!(finish_reason_for("content_filter"), "content_filter");
+    }
+
+    #[test]
+    fn defaults_an_unmapped_reason_to_stop_like_python() {
+        // Python warns and falls back to `stop` for a reason its own map does
+        // not carry, so only a reason absent from `_FINISH_REASON_MAP` belongs
+        // here.
+        assert_eq!(finish_reason_for("something_new"), "stop");
+        assert_eq!(finish_reason_for(""), "stop");
+    }
+
+    #[test]
+    fn folds_cache_tokens_into_prompt_tokens() {
+        let usage = usage_from_parts(10, 4, 7, 3);
+        assert_eq!(usage.prompt_tokens, 20);
+        assert_eq!(usage.completion_tokens, 4);
+        assert_eq!(usage.total_tokens, 24);
+        assert_eq!(usage.prompt_tokens_details.cached_tokens, 7);
+        assert_eq!(usage.prompt_tokens_details.cache_creation_tokens, 3);
+        assert_eq!(usage.prompt_tokens_details.text_tokens, 10);
+    }
+
+    #[test]
+    fn reports_raw_input_tokens_when_no_cache_is_involved() {
+        let usage = usage_from_parts(12, 5, 0, 0);
+        assert_eq!(usage.prompt_tokens, 12);
+        assert_eq!(usage.total_tokens, 17);
+        assert_eq!(usage.prompt_tokens_details.text_tokens, 12);
+    }
+}

litellm-rust/crates/core/src/chat_completions/tests.rs

@@ -0,0 +1,820 @@
+use serde_json::{Map, Value, json};
+
+use crate::error::CoreError;
+
+use super::prepare::prepare_chat_completions_call;
+use super::transformation::ChatCompletionsAuth;
+use super::types::ChatCompletionsRequest;
+
+fn request<'a>(
+    model: &'a str,
+    provider: Option<&'a str>,
+    messages: Value,
+    optional_params: Value,
+) -> ChatCompletionsRequest<'a> {
+    ChatCompletionsRequest {
+        model,
+        messages,
+        optional_params: match optional_params {
+            Value::Object(map) => map,
+            other => panic!("params must be an object, got {other}"),
+        },
+        api_key: Some("sk-test"),
+        api_base: None,
+        custom_llm_provider: provider,
+        extra_headers: None,
+        timeout: None,
+    }
+}
+
+/// `ProviderChatCompletionsRequest` deliberately has no `Debug` (its headers
+/// carry resolved credentials), so unwrap the failure case by hand.
+fn decline(request: ChatCompletionsRequest<'_>) -> CoreError {
+    match prepare_chat_completions_call(request) {
+        Err(error) => error,
+        Ok(prepared) => panic!("expected a decline, prepared a call to {}", prepared.url),
+    }
+}
+
+#[test]
+fn resolves_the_provider_from_the_model_prefix() {
+    let prepared = prepare_chat_completions_call(request(
+        "anthropic/claude-sonnet-4-5",
+        None,
+        json!([{"role": "user", "content": "hi"}]),
+        json!({"max_tokens": 16}),
+    ))
+    .expect("prepares");
+    assert_eq!(prepared.model, "claude-sonnet-4-5");
+    assert_eq!(prepared.url, "https://api.anthropic.com/v1/messages");
+    assert_eq!(prepared.body["model"], json!("claude-sonnet-4-5"));
+}
+
+#[test]
+fn strips_an_explicit_provider_prefix_from_the_model() {
+    let prepared = prepare_chat_completions_call(request(
+        "anthropic/claude-sonnet-4-5",
+        Some("anthropic"),
+        json!([{"role": "user", "content": "hi"}]),
+        json!({}),
+    ))
+    .expect("prepares");
+    assert_eq!(prepared.model, "claude-sonnet-4-5");
+}
+
+#[test]
+fn adds_the_auth_and_default_headers() {
+    let prepared = prepare_chat_completions_call(request(
+        "claude-sonnet-4-5",
+        Some("anthropic"),
+        json!([{"role": "user", "content": "hi"}]),
+        json!({}),
+    ))
+    .expect("prepares");
+    assert!(
+        prepared
+            .upstream_headers
+            .contains(&("x-api-key".to_string(), "sk-test".to_string()))
+    );
+    assert!(
+        prepared
+            .upstream_headers
+            .contains(&("anthropic-version".to_string(), "2023-06-01".to_string()))
+    );
+    assert!(matches!(
+        prepared.auth,
+        ChatCompletionsAuth::Header {
+            name: "x-api-key",
+            ..
+        }
+    ));
+}
+
+#[test]
+fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
+    // Python builds `{**headers, **anthropic_headers}`, so the deployment's key
+    // overwrites a forwarded one. Honouring the caller's would let whoever sends
+    // the request choose the Anthropic principal it bills to.
+    let mut call = request(
+        "claude-sonnet-4-5",
+        Some("anthropic"),
+        json!([{"role": "user", "content": "hi"}]),
+        json!({}),
+    );
+    call.extra_headers = Some(Map::from_iter([(
+        "X-Api-Key".to_string(),
+        json!("sk-caller"),
+    )]));
+    let prepared = prepare_chat_completions_call(call).expect("prepares");
+    let keys: Vec<_> = prepared
+        .upstream_headers
+        .iter()
+        .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
+        .collect();
+    assert_eq!(keys.len(), 1, "got {:?}", prepared.upstream_headers);
+    assert_eq!(keys[0].1, "sk-test");
+}
+
+#[test]
+fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
+    // Anthropic's `validate_environment` pops `x-api-key` and sets `authorization`
+    // for an OAuth token, so re-adding the key here would put the credential into
+    // a header the host removed on purpose.
+    let mut call = request(
+        "claude-sonnet-4-5",
+        Some("anthropic"),
+        json!([{"role": "user", "content": "hi"}]),
+        json!({}),
+    );
+    call.extra_headers = Some(Map::from_iter([
+        (
+            "Authorization".to_string(),
+            json!("Bearer sk-ant-oat01-token"),
+        ),
+        ("X-Api-Key".to_string(), json!("sk-caller")),
+    ]));
+    let prepared = prepare_chat_completions_call(call).expect("prepares");
+    assert!(
+        !prepared
+            .upstream_headers
+            .iter()
+            .any(|(name, value)| name.eq_ignore_ascii_case("x-api-key") && value == "sk-test"),
+        "the resolved key must not be applied over an OAuth bearer, got {:?}",
+        prepared.upstream_headers
+    );
+    assert!(
+        prepared
+            .upstream_headers
+            .iter()
+            .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
+                && value == "Bearer sk-ant-oat01-token")
+    );
+}
+
+#[test]
+fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
+    // Only an OAuth bearer replaces the credential. Python sends the deployment's
+    // `x-api-key` alongside any other forwarded `authorization`, so deferring on
+    // the mere presence of that header would drop the deployment's auth.
+    let mut call = request(
+        "claude-sonnet-4-5",
+        Some("anthropic"),
+        json!([{"role": "user", "content": "hi"}]),
+        json!({}),
+    );
+    call.extra_headers = Some(Map::from_iter([
+        ("Authorization".to_string(), json!("Bearer unrelated")),
+        ("X-Api-Key".to_string(), json!("sk-caller")),
+    ]));
+    let prepared = prepare_chat_completions_call(call).expect("prepares");
+    let keys: Vec<_> = prepared
+        .upstream_headers
+        .iter()
+        .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
+        .collect();
+    assert_eq!(keys.len(), 1, "got {:?}", prepared.upstream_headers);
+    assert_eq!(keys[0].1, "sk-test");
+    assert!(
+        prepared
+            .upstream_headers
+            .iter()
+            .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
+                && value == "Bearer unrelated"),
+        "the unrelated authorization must survive, got {:?}",
+        prepared.upstream_headers
+    );
+}
+
+#[test]
+fn declines_an_unsupported_request_before_resolving_credentials() {
+    let mut call = request(
+        "claude-sonnet-4-5",
+        Some("anthropic"),
+        json!([{"role": "user", "content": "hi"}]),
+        json!({"stream": true}),
+    );
+    call.api_key = None;
+    // No api_key is set and no env is consulted: the gate must run first, so the
+    // error is the decline rather than a missing-credential error.
+    assert_eq!(decline(call), CoreError::Unsupported("streaming"));
+}
+
+#[test]
+fn rejects_an_unknown_provider() {
+    assert_eq!(
+        decline(request(
+            "openai/gpt-4o",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )),
+        CoreError::InvalidProvider("openai".to_string())
+    );
+}
+
+#[test]
+fn rejects_a_model_with_no_resolvable_provider() {
+    assert!(matches!(
+        decline(request(
+            "claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )),
+        CoreError::InvalidProvider(_)
+    ));
+}
+
+#[test]
+fn rejects_an_empty_or_malformed_message_list() {
+    assert_eq!(
+        decline(request(
+            "anthropic/claude-sonnet-4-5",
+            None,
+            json!([]),
+            json!({}),
+        )),
+        CoreError::InvalidRequest("chat completions requires at least one message".to_string())
+    );
+    assert!(matches!(
+        decline(request(
+            "anthropic/claude-sonnet-4-5",
+            None,
+            json!("not a list"),
+            json!({}),
+        )),
+        CoreError::InvalidRequest(_)
+    ));
+}
+
+#[test]
+fn rejects_non_string_extra_headers() {
+    let mut call = request(
+        "anthropic/claude-sonnet-4-5",
+        None,
+        json!([{"role": "user", "content": "hi"}]),
+        json!({}),
+    );
+    call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))]));
+    assert_eq!(
+        decline(call),
+        CoreError::InvalidRequest(
+            "chat completions extra_headers.x-trace must be a string, got number".to_string()
+        )
+    );
+}
+
+#[cfg(feature = "bedrock-auth")]
+#[test]
+fn prepares_a_bedrock_call_without_resolving_credentials() {
+    let mut call = request(
+        "bedrock/us-east-1/anthropic.claude-v2",
+        None,
+        json!([{"role": "user", "content": "hi"}]),
+        json!({"maxTokens": 16}),
+    );
+    call.api_key = None;
+    let prepared = prepare_chat_completions_call(call).expect("prepares");
+    assert_eq!(
+        prepared.url,
+        "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
+    );
+    assert_eq!(
+        prepared.auth,
+        ChatCompletionsAuth::AwsSigV4 {
+            region: "us-east-1".to_string()
+        }
+    );
+    // SigV4 signs the serialized body, so prepare must not have added an
+    // Authorization header; the handler does it.
+    assert!(
+        !prepared
+            .upstream_headers
+            .iter()
+            .any(|(name, _)| name.eq_ignore_ascii_case("authorization"))
+    );
+    assert_eq!(prepared.body["inferenceConfig"], json!({"maxTokens": 16}));
+}
+
+#[cfg(feature = "bedrock-auth")]
+#[tokio::test]
+async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
+    // Python signs only the AWS header set and reattaches the rest, so a header
+    // the caller forwarded rides along without joining the canonical request.
+    // Signing it makes Converse 403 on a deployment that works on Python.
+    let mut call = request(
+        "bedrock/us-east-1/anthropic.claude-v2",
+        None,
+        json!([{"role": "user", "content": "hi"}]),
+        json!({
+            "maxTokens": 16,
+            "aws_access_key_id": "AKIDEXAMPLE",
+            "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
+        }),
+    );
+    // A key would resolve to a bearer token and never reach the signer.
+    call.api_key = None;
+    call.extra_headers = Some(Map::from_iter([(
+        "x-request-id".to_string(),
+        json!("abc-123"),
+    )]));
+    let prepared = prepare_chat_completions_call(call).expect("prepares");
+    let signed = super::handler::signed_headers(&prepared, br#"{"a":1}"#)
+        .await
+        .expect("signs");
+
+    let authorization = signed
+        .iter()
+        .find(|(name, _)| name.eq_ignore_ascii_case("authorization"))
+        .map(|(_, value)| value.clone())
+        .expect("carries an authorization header");
+    assert!(
+        authorization.starts_with("AWS4-HMAC-SHA256"),
+        "expected a SigV4 signature, got {authorization}"
+    );
+    assert!(
+        !authorization.contains("x-request-id"),
+        "forwarded header reached SignedHeaders: {authorization}"
+    );
+    // It still goes on the wire, it is just not part of the signature.
+    assert!(
+        signed
+            .iter()
+            .any(|(name, value)| name == "x-request-id" && value == "abc-123"),
+        "forwarded header was dropped instead of reattached"
+    );
+}
+
+#[cfg(feature = "bedrock-auth")]
+#[tokio::test]
+async fn a_forwarded_header_the_signer_computes_declines_to_python() {
+    // Reattaching the caller's copy next to the computed one puts the name on
+    // the wire twice and Bedrock rejects the pair, so a request carrying one
+    // has to go to Python instead of being signed here.
+    for forwarded in [
+        "Authorization",
+        "x-amz-date",
+        "x-amz-security-token",
+        "Date",
+    ] {
+        let mut call = request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({
+                "maxTokens": 16,
+                "aws_access_key_id": "AKIDEXAMPLE",
+                "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
+            }),
+        );
+        call.api_key = None;
+        call.extra_headers = Some(Map::from_iter([(forwarded.to_string(), json!("forged"))]));
+        let prepared = prepare_chat_completions_call(call).expect("prepares");
+        let error = super::handler::signed_headers(&prepared, br#"{"a":1}"#)
+            .await
+            .expect_err("{forwarded} should decline instead of being signed");
+        assert!(
+            matches!(error, CoreError::Unsupported(_)),
+            "{forwarded} declined as {error:?}, which the host would not fall back on"
+        );
+    }
+}
+
+#[cfg(feature = "bedrock-auth")]
+#[test]
+fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
+    // `get_request_headers` assigns `headers["Authorization"]` unconditionally
+    // once a bearer token resolves, so the deployment's identity wins on
+    // Python. Keeping the caller's would authorize and bill the call as a
+    // different principal, and only when the deployment carries `rust: true`.
+    let mut call = request(
+        "bedrock/us-east-1/anthropic.claude-v2",
+        None,
+        json!([{"role": "user", "content": "hi"}]),
+        json!({"maxTokens": 16}),
+    );
+    call.extra_headers = Some(Map::from_iter([(
+        "Authorization".to_string(),
+        json!("Bearer caller-supplied"),
+    )]));
+    let prepared = prepare_chat_completions_call(call).expect("prepares");
+    let authorizations: Vec<_> = prepared
+        .upstream_headers
+        .iter()
+        .filter(|(name, _)| name.eq_ignore_ascii_case("authorization"))
+        .map(|(_, value)| value.as_str())
+        .collect();
+    assert_eq!(
+        authorizations,
+        vec!["Bearer sk-test"],
+        "the deployment token must be the only authorization on the wire"
+    );
+}
+
+#[test]
+fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
+    // The opposite precedence, and deliberate: Anthropic's own transform
+    // honours a forwarded OAuth bearer, so the Bedrock fix above must not be
+    // generalized into a rule that the configured key always wins.
+    //
+    // An OAuth bearer is the whole of that exception. This forwarded a plain
+    // `x-api-key` until round 17, which read as the same claim and was not:
+    // Python overwrites a forwarded `x-api-key` with the deployment's.
+    let mut call = request(
+        "claude-sonnet-4-5",
+        Some("anthropic"),
+        json!([{"role": "user", "content": "hi"}]),
+        json!({}),
+    );
+    call.extra_headers = Some(Map::from_iter([(
+        "authorization".to_string(),
+        json!("Bearer sk-ant-oat01-forwarded"),
+    )]));
+    let prepared = prepare_chat_completions_call(call).expect("prepares");
+    let keys: Vec<_> = prepared
+        .upstream_headers
+        .iter()
+        .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
+        .map(|(_, value)| value.as_str())
+        .collect();
+    assert!(keys.is_empty(), "got {:?}", prepared.upstream_headers);
+    assert!(
+        prepared
+            .upstream_headers
+            .iter()
+            .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
+                && value == "Bearer sk-ant-oat01-forwarded")
+    );
+}
+
+#[cfg(feature = "bedrock-auth")]
+#[test]
+fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
+    // The configured bearer identity has its own account and quota boundary,
+    // so a request carrying one must not be signed as whatever principal the
+    // host's AWS credentials resolve to.
+    let prepared = prepare_chat_completions_call(request(
+        "bedrock/us-east-1/anthropic.claude-v2",
+        None,
+        json!([{"role": "user", "content": "hi"}]),
+        json!({"maxTokens": 16}),
+    ))
+    .expect("prepares");
+    assert_eq!(
+        prepared.auth,
+        ChatCompletionsAuth::Bearer {
+            token: "sk-test".to_string()
+        }
+    );
+    assert!(
+        prepared
+            .upstream_headers
+            .iter()
+            .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
+                && value == "Bearer sk-test"),
+        "prepare did not carry the bearer token"
+    );
+}
+
+fn decline_reason(
+    model: &str,
+    provider: Option<&str>,
+    messages: Value,
+    params: Value,
+) -> Option<&'static str> {
+    let params = match params {
+        Value::Object(map) => map,
+        other => panic!("params must be an object, got {other}"),
+    };
+    super::chat_completions_decline_reason(model, provider, messages, &params)
+}
+
+#[test]
+fn the_gate_accepts_what_prepare_accepts() {
+    assert_eq!(
+        decline_reason(
+            "anthropic/claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"max_tokens": 16}),
+        ),
+        None
+    );
+}
+
+#[test]
+fn the_gate_declines_without_resolving_credentials_or_calling_out() {
+    assert_eq!(
+        decline_reason(
+            "anthropic/claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"stream": true}),
+        ),
+        Some("streaming")
+    );
+    assert_eq!(
+        decline_reason(
+            "openai/gpt-4o",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        ),
+        Some("provider is not on the rust chat completions path")
+    );
+    assert_eq!(
+        decline_reason(
+            "claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        ),
+        Some("provider is not on the rust chat completions path")
+    );
+    assert_eq!(
+        decline_reason(
+            "anthropic/claude-sonnet-4-5",
+            None,
+            json!("nope"),
+            json!({})
+        ),
+        Some("unreadable message list")
+    );
+    assert_eq!(
+        decline_reason("anthropic/claude-sonnet-4-5", None, json!([]), json!({})),
+        Some("empty message list")
+    );
+}
+
+#[test]
+fn the_gate_agrees_with_prepare_on_every_case_it_accepts() {
+    // A gate that accepts what prepare then declines would make the host emit
+    // its pre-call logging on a path that falls back, so pin the agreement.
+    for (messages, params) in [
+        (
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"max_tokens": 8}),
+        ),
+        (
+            json!([{"role": "system", "content": "s"}, {"role": "user", "content": "hi"}]),
+            json!({"temperature": 0.1}),
+        ),
+        (
+            json!([{"role": "user", "content": "hi"}, {"role": "assistant", "content": "yo"}]),
+            json!({}),
+        ),
+    ] {
+        assert_eq!(
+            decline_reason(
+                "anthropic/claude-sonnet-4-5",
+                None,
+                messages.clone(),
+                params.clone()
+            ),
+            None,
+            "gate declined {messages}"
+        );
+        prepare_chat_completions_call(request(
+            "anthropic/claude-sonnet-4-5",
+            None,
+            messages.clone(),
+            params,
+        ))
+        .unwrap_or_else(|error| panic!("prepare declined {messages}: {error}"));
+    }
+}
+
+mod round_trip {
+    use super::*;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::{TcpListener, TcpStream};
+
+    use crate::chat_completions::chat_completions;
+
+    async fn read_http_request(socket: &mut TcpStream) -> String {
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 1024];
+        let header_end = loop {
+            let n = socket.read(&mut buffer).await.expect("reads request");
+            if n == 0 {
+                break request.len();
+            }
+            request.extend_from_slice(&buffer[..n]);
+            if let Some(position) = request.windows(4).position(|window| window == b"\r\n\r\n") {
+                break position + 4;
+            }
+        };
+        let headers = String::from_utf8_lossy(&request[..header_end]);
+        let content_length = headers
+            .lines()
+            .find_map(|line| {
+                let (name, value) = line.split_once(':')?;
+                name.eq_ignore_ascii_case("content-length")
+                    .then(|| value.trim().parse::<usize>().ok())
+                    .flatten()
+            })
+            .unwrap_or(0);
+        while request.len().saturating_sub(header_end) < content_length {
+            let n = socket.read(&mut buffer).await.expect("reads body");
+            if n == 0 {
+                break;
+            }
+            request.extend_from_slice(&buffer[..n]);
+        }
+        String::from_utf8(request).expect("request is utf8")
+    }
+
+    fn http_response(status: &str, body: &str) -> String {
+        format!(
+            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+            body.len(),
+            body
+        )
+    }
+
+    /// Serve one request from a stub upstream and hand back what it received.
+    async fn serve_once(
+        status: &'static str,
+        body: &'static str,
+    ) -> (String, tokio::task::JoinHandle<String>) {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+        let port = listener.local_addr().expect("addr").port();
+        let handle = tokio::spawn(async move {
+            let (mut socket, _) = listener.accept().await.expect("accepts");
+            let received = read_http_request(&mut socket).await;
+            socket
+                .write_all(http_response(status, body).as_bytes())
+                .await
+                .expect("writes response");
+            socket.flush().await.expect("flushes");
+            received
+        });
+        (format!("http://127.0.0.1:{port}/v1/messages"), handle)
+    }
+
+    fn call(api_base: &str, messages: Value, params: Value) -> ChatCompletionsRequest<'_> {
+        ChatCompletionsRequest {
+            model: "anthropic/claude-sonnet-4-5",
+            messages,
+            optional_params: match params {
+                Value::Object(map) => map,
+                other => panic!("params must be an object, got {other}"),
+            },
+            api_key: Some("sk-test"),
+            api_base: Some(api_base),
+            custom_llm_provider: None,
+            extra_headers: None,
+            timeout: Some(std::time::Duration::from_secs(10)),
+        }
+    }
+
+    const GOOD_BODY: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5-20260101","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
+
+    #[tokio::test]
+    async fn round_trip_sends_the_translated_body_and_normalizes_the_response() {
+        let (api_base, handle) = serve_once("200 OK", GOOD_BODY).await;
+        let response = chat_completions(call(
+            &api_base,
+            json!([
+                {"role": "system", "content": "be terse"},
+                {"role": "user", "content": "hi"}
+            ]),
+            json!({"max_tokens": 16}),
+        ))
+        .await
+        .expect("call succeeds");
+
+        let received = handle.await.expect("server task");
+        let sent: Value = serde_json::from_str(
+            received
+                .split_once("\r\n\r\n")
+                .expect("request has a body")
+                .1,
+        )
+        .expect("body is json");
+        assert_eq!(
+            sent["messages"],
+            json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}])
+        );
+        assert_eq!(
+            sent["system"],
+            json!([{"type": "text", "text": "be terse"}])
+        );
+        assert_eq!(sent["max_tokens"], json!(16));
+        assert!(received.to_lowercase().contains("x-api-key: sk-test"));
+
+        assert_eq!(
+            response.choices[0].message.content.as_deref(),
+            Some("hello")
+        );
+        assert_eq!(response.usage.total_tokens, 15);
+    }
+
+    #[tokio::test]
+    async fn a_response_it_cannot_normalize_is_reported_as_already_sent() {
+        // The provider was called and billed, so the host must not retry this
+        // on its own path. `MissingField` here would read as a pre-send
+        // decline and be retried; `InvalidResponse` cannot.
+        const NO_USAGE: &str =
+            r#"{"model":"m","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn"}"#;
+        let (api_base, handle) = serve_once("200 OK", NO_USAGE).await;
+        let err = chat_completions(call(
+            &api_base,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"max_tokens": 16}),
+        ))
+        .await
+        .expect_err("response cannot be normalized");
+        handle.await.expect("server task");
+        assert!(
+            matches!(err, CoreError::InvalidResponse(_)),
+            "expected a post-send error, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn a_tool_use_block_in_the_response_is_also_reported_as_already_sent() {
+        const TOOL_USE: &str = r#"{"model":"m","content":[{"type":"tool_use","id":"t","name":"f","input":{}}],"stop_reason":"tool_use","usage":{"input_tokens":1,"output_tokens":1}}"#;
+        let (api_base, handle) = serve_once("200 OK", TOOL_USE).await;
+        let err = chat_completions(call(
+            &api_base,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"max_tokens": 16}),
+        ))
+        .await
+        .expect_err("response cannot be normalized");
+        handle.await.expect("server task");
+        assert!(
+            matches!(err, CoreError::InvalidResponse(_)),
+            "expected a post-send error, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn an_upstream_error_status_keeps_its_code() {
+        let (api_base, handle) =
+            serve_once("429 Too Many Requests", r#"{"error":"slow down"}"#).await;
+        let err = chat_completions(call(
+            &api_base,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"max_tokens": 16}),
+        ))
+        .await
+        .expect_err("upstream rejects");
+        handle.await.expect("server task");
+        assert!(
+            matches!(err, CoreError::Http { status: 429, .. }),
+            "expected a 429, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn a_connection_that_is_never_established_declines_instead_of_failing() {
+        // Nothing was sent, so nothing was billed and the host can still serve
+        // the request. Classing this with the post-send failures would turn a
+        // recoverable fallback into a user-facing error on exactly the
+        // deployments whose transport is configured only on the Python client.
+        let port = {
+            let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+            listener.local_addr().expect("has an address").port()
+            // Dropped here, so the port is closed and the connect is refused.
+        };
+        let err = chat_completions(call(
+            &format!("http://127.0.0.1:{port}/v1/messages"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"max_tokens": 16}),
+        ))
+        .await
+        .expect_err("nothing is listening");
+        assert!(
+            matches!(err, CoreError::Connect(_)),
+            "expected a pre-send connect failure, got {err:?}"
+        );
+    }
+
+    #[test]
+    fn response_errors_collapse_to_one_variant_that_can_only_mean_already_sent() {
+        use crate::chat_completions::handler::as_response_error;
+
+        for original in [
+            CoreError::MissingField("usage"),
+            CoreError::Unsupported("non-text response content block"),
+            CoreError::InvalidRequest("whatever".to_string()),
+            CoreError::Auth("whatever".to_string()),
+        ] {
+            let label = format!("{original:?}");
+            assert!(
+                matches!(as_response_error(original), CoreError::InvalidResponse(_)),
+                "{label} must not stay retryable once the provider has answered"
+            );
+        }
+        // An upstream status is already unambiguous, so it survives intact.
+        assert!(matches!(
+            as_response_error(CoreError::Http {
+                status: 500,
+                body: "boom".to_string()
+            }),
+            CoreError::Http { status: 500, .. }
+        ));
+    }
+}

litellm-rust/crates/core/src/chat_completions/transformation.rs

@@ -0,0 +1,155 @@
+use serde_json::{Map, Value};
+
+use crate::error::CoreResult;
+
+use super::types::{
+    ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData,
+    ProviderChatResponseData,
+};
+
+/// How the upstream call is authenticated. API-key strategies are resolved in
+/// `prepare`; SigV4 needs the serialized body, so the handler signs it.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ChatCompletionsAuth {
+    Header { name: &'static str, value: String },
+    Bearer { token: String },
+    AwsSigV4 { region: String },
+}
+
+/// Why a request cannot be served by the Rust path.
+///
+/// The core declines rather than guessing: the host turns this into a
+/// transparent fallback to the Python implementation, which covers the full
+/// surface. Acceptance is an allowlist, so a parameter or message shape the
+/// core has never seen declines by construction instead of being translated
+/// wrong.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Unsupported(pub &'static str);
+
+pub const STREAM_PARAM: &str = "stream";
+
+/// Message fields that carry no meaning for the upstream body, so their
+/// presence does not make a request untranslatable.
+const IGNORABLE_MESSAGE_FIELDS: &[&str] = &["name"];
+
+pub trait ChatCompletionsProviderConfig: Sync {
+    fn complete_url(
+        &self,
+        api_base: Option<&str>,
+        model: &str,
+        optional_params: &Map<String, Value>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<String>;
+
+    fn auth(
+        &self,
+        api_key: Option<&str>,
+        model: &str,
+        optional_params: &Map<String, Value>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<ChatCompletionsAuth>;
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        &[("content-type", "application/json")]
+    }
+
+    /// Whether an auth header the caller already supplied is the credential this
+    /// request should authenticate with, so the resolved one is not applied.
+    ///
+    /// Defaults to false: the deployment's credential outranks anything
+    /// forwarded, which is what every provider wants for its own auth header.
+    /// A provider overrides this only for a scheme it hands off to entirely.
+    fn defers_to_forwarded_auth(&self, _headers: &[(String, String)]) -> bool {
+        false
+    }
+
+    /// Provider parameter names (post-mapping) the Rust path knows how to place
+    /// in the upstream body. Anything outside this set declines the request.
+    fn supported_params(&self) -> &'static [&'static str];
+
+    /// Parameters consumed as call configuration (credentials, endpoints)
+    /// rather than placed in the body. Accepted, never serialized.
+    fn config_params(&self) -> &'static [&'static str] {
+        &[]
+    }
+
+    fn unsupported_reason(
+        &self,
+        messages: &[ChatMessage],
+        optional_params: &Map<String, Value>,
+    ) -> Option<Unsupported> {
+        unsupported_param(
+            self.supported_params(),
+            self.config_params(),
+            optional_params,
+        )
+        .or_else(|| messages.iter().find_map(unsupported_message))
+    }
+
+    fn transform_request(
+        &self,
+        model: &str,
+        messages: Vec<ChatMessage>,
+        optional_params: Map<String, Value>,
+    ) -> CoreResult<ProviderChatRequestData>;
+
+    fn transform_response(
+        &self,
+        model: &str,
+        response: ProviderChatResponseData,
+    ) -> CoreResult<ChatCompletionsResponse>;
+}
+
+pub fn unsupported_param(
+    supported: &'static [&'static str],
+    config: &'static [&'static str],
+    optional_params: &Map<String, Value>,
+) -> Option<Unsupported> {
+    if optional_params
+        .get(STREAM_PARAM)
+        .and_then(Value::as_bool)
+        .unwrap_or(false)
+    {
+        return Some(Unsupported("streaming"));
+    }
+    optional_params
+        .keys()
+        .any(|key| {
+            key != STREAM_PARAM
+                && !supported.contains(&key.as_str())
+                && !config.contains(&key.as_str())
+        })
+        .then_some(Unsupported("unrecognized request parameter"))
+}
+
+/// Message shapes the core can translate faithfully: text content, either a
+/// plain string or a non-empty list of parts that are all
+/// `{"type": "text", "text": ...}`. Tool calls, tool results, and multimodal
+/// parts decline so Python's fuller translation handles them.
+pub fn unsupported_message(message: &ChatMessage) -> Option<Unsupported> {
+    if message
+        .extra
+        .keys()
+        .any(|key| !IGNORABLE_MESSAGE_FIELDS.contains(&key.as_str()))
+    {
+        return Some(Unsupported("unrecognized message field"));
+    }
+    if !matches!(message.role.as_str(), "system" | "user" | "assistant") {
+        return Some(Unsupported("unrecognized message role"));
+    }
+    match &message.content {
+        None => Some(Unsupported("message without content")),
+        Some(ChatMessageContent::Text(_)) => None,
+        Some(ChatMessageContent::Parts(parts)) if parts.is_empty() => {
+            Some(Unsupported("message without content"))
+        }
+        Some(ChatMessageContent::Parts(parts)) => parts
+            .iter()
+            .any(|part| {
+                part.get("type").and_then(Value::as_str) != Some("text")
+                    || part.get("text").and_then(Value::as_str).is_none()
+                    || part.as_object().is_some_and(|object| object.len() != 2)
+            })
+            .then_some(Unsupported("non-text message content")),
+    }
+}

litellm-rust/crates/core/src/chat_completions/types.rs

@@ -0,0 +1,112 @@
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
+
+use super::transformation::{ChatCompletionsAuth, ChatCompletionsProviderConfig};
+
+/// A `/chat/completions` call as it crosses into the core.
+///
+/// `optional_params` arrives already mapped to the provider's own parameter
+/// names by the host, exactly as the messages route receives an already
+/// Anthropic-shaped body. The core owns the conversation translation, the
+/// provider call, and the response normalization.
+pub struct ChatCompletionsRequest<'a> {
+    pub model: &'a str,
+    pub messages: Value,
+    pub optional_params: Map<String, Value>,
+    pub api_key: Option<&'a str>,
+    pub api_base: Option<&'a str>,
+    pub custom_llm_provider: Option<&'a str>,
+    pub extra_headers: Option<Map<String, Value>>,
+    pub timeout: Option<Duration>,
+}
+
+pub(super) struct ProviderChatCompletionsRequest {
+    pub(super) model: String,
+    pub(super) config: &'static dyn ChatCompletionsProviderConfig,
+    pub(super) url: String,
+    pub(super) body: Value,
+    pub(super) upstream_headers: Vec<(String, String)>,
+    pub(super) auth: ChatCompletionsAuth,
+    #[cfg_attr(not(feature = "bedrock-auth"), allow(dead_code))]
+    pub(super) optional_params: Map<String, Value>,
+    pub(super) timeout: Option<Duration>,
+}
+
+/// The provider-shaped request body a config produces. Named rather than a bare
+/// `Value` so the transform contract stays a typed one, mirroring
+/// [`crate::audio_transcription::types::AudioTranscriptionRequestData`].
+pub struct ProviderChatRequestData {
+    pub body: Value,
+}
+
+/// The raw provider response body handed back to a config for normalization.
+pub struct ProviderChatResponseData {
+    pub body: Value,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum ChatMessageContent {
+    Text(String),
+    Parts(Vec<Value>),
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct ChatMessage {
+    pub role: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub content: Option<ChatMessageContent>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub name: Option<String>,
+    #[serde(flatten)]
+    pub extra: Map<String, Value>,
+}
+
+/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python
+/// path reports so cost tracking sees the same numbers on either path.
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+pub struct PromptTokensDetails {
+    pub cached_tokens: u64,
+    pub cache_creation_tokens: u64,
+    pub text_tokens: u64,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+pub struct ChatCompletionsUsage {
+    pub prompt_tokens: u64,
+    pub completion_tokens: u64,
+    pub total_tokens: u64,
+    pub prompt_tokens_details: PromptTokensDetails,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct ChatCompletionsChoiceMessage {
+    pub role: String,
+    // Whether an empty turn is `None` or `""` is the provider's choice, not a
+    // shared invariant: Anthropic's transform ends on `merged_text or None`
+    // while Converse assigns the joined string unconditionally. Each config
+    // mirrors its own, so keep this optional and serialize it even when None.
+    pub content: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct ChatCompletionsChoice {
+    pub index: u64,
+    pub message: ChatCompletionsChoiceMessage,
+    pub finish_reason: String,
+}
+
+/// The normalized response handed back to the host.
+///
+/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the
+/// `ModelResponse` it already created, and echoing the provider's own id here
+/// would change it. Pinned by `response_carries_no_id` in `tests.rs`.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct ChatCompletionsResponse {
+    pub created: u64,
+    pub model: String,
+    pub choices: Vec<ChatCompletionsChoice>,
+    pub usage: ChatCompletionsUsage,
+}

litellm-rust/crates/core/src/constants.rs

@@ -12,8 +12,30 @@ pub(crate) const MESSAGES_CONNECT_TIMEOUT_SECS: u64 = 10;
 
 /// Max characters of an upstream error body echoed across the call boundary
 /// before truncation, so provider bodies are bounded and data-minimized.
-pub(crate) const MESSAGES_ERROR_BODY_MAX_CHARS: usize = 256;
+pub(crate) const UPSTREAM_ERROR_BODY_MAX_CHARS: usize = 256;
 
 /// Provider name used for Anthropic Messages when a deployment's provider model
 /// does not carry an explicit provider prefix.
 pub const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic";
+
+/// Prefix identifying an Anthropic OAuth token. Mirrors Python's
+/// `ANTHROPIC_OAUTH_TOKEN_PREFIX`, which is what makes `validate_environment`
+/// authenticate with `authorization` and drop `x-api-key` entirely.
+pub(crate) const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat";
+
+/// Full-request timeout ceiling for chat completions provider calls, in
+/// seconds. Mirrors the Python chat completions default.
+pub(crate) const CHAT_COMPLETIONS_TIMEOUT_SECS: u64 = 600;
+
+/// Connect timeout for chat completions provider calls, in seconds.
+pub(crate) const CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS: u64 = 10;
+
+/// `object` field every non-streaming chat completion response carries.
+pub const CHAT_COMPLETION_OBJECT: &str = "chat.completion";
+
+/// Placeholder Python substitutes for empty or whitespace-only message text,
+/// which Anthropic and Bedrock both reject. Must match
+/// `_EMPTY_TEXT_PLACEHOLDER` in
+/// `litellm/litellm_core_utils/prompt_templates/factory.py`.
+pub const EMPTY_TEXT_PLACEHOLDER: &str =
+    "[System: Empty message content sanitised to satisfy protocol]";

litellm-rust/crates/core/src/error.rs

@@ -23,8 +23,19 @@ pub enum CoreError {
     Http { status: u16, body: String },
     #[error("upstream network error: {0}")]
     Network(String),
+    /// The provider was never reached: DNS, TCP, TLS or proxy setup failed
+    /// before any byte of the request went out. Nothing was billed, so a host
+    /// that keeps a reference implementation can serve the request itself.
+    /// A timeout is deliberately not this, since the provider may have received
+    /// and answered the request already.
+    #[error("could not reach the provider: {0}")]
+    Connect(String),
     #[error("routing error: {0}")]
     Routing(String),
+    /// The request is outside the surface this route covers in Rust. Hosts that
+    /// keep a reference implementation treat this as "fall back", not "fail".
+    #[error("unsupported by the rust path: {0}")]
+    Unsupported(&'static str),
 }
 
 pub fn json_type_name(value: &serde_json::Value) -> &'static str {

litellm-rust/crates/core/src/http_utils.rs

@@ -0,0 +1,112 @@
+//! Header and upstream-body helpers shared by every route module.
+
+use serde_json::{Map, Value};
+
+use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS;
+use crate::error::{CoreError, CoreResult, json_type_name};
+
+/// Bound an upstream error body before it crosses a host boundary, so provider
+/// bodies stay data-minimized.
+pub fn truncate_error_body(body: &str) -> String {
+    if body.chars().count() <= UPSTREAM_ERROR_BODY_MAX_CHARS {
+        return body.to_string();
+    }
+    let truncated: String = body.chars().take(UPSTREAM_ERROR_BODY_MAX_CHARS).collect();
+    format!("{truncated}... (truncated)")
+}
+
+pub fn string_headers(
+    context: &'static str,
+    extra_headers: Option<Map<String, Value>>,
+) -> CoreResult<Vec<(String, String)>> {
+    extra_headers
+        .unwrap_or_default()
+        .into_iter()
+        .map(|(key, value)| {
+            value
+                .as_str()
+                .map(|value| (key.clone(), value.to_string()))
+                .ok_or_else(|| {
+                    CoreError::InvalidRequest(format!(
+                        "{context} extra_headers.{key} must be a string, got {}",
+                        json_type_name(&value)
+                    ))
+                })
+        })
+        .collect()
+}
+
+pub fn has_header(headers: &[(String, String)], name: &str) -> bool {
+    headers
+        .iter()
+        .any(|(key, _)| key.eq_ignore_ascii_case(name))
+}
+
+pub fn has_bearer_auth(headers: &[(String, String)]) -> bool {
+    headers.iter().any(|(name, value)| {
+        if !name.eq_ignore_ascii_case("authorization") {
+            return false;
+        }
+        let value = value.trim();
+        value.len() > 7
+            && value[..7].eq_ignore_ascii_case("bearer ")
+            && !value[7..].trim().is_empty()
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn truncate_leaves_short_bodies_untouched() {
+        assert_eq!(truncate_error_body("short"), "short");
+    }
+
+    #[test]
+    fn truncate_bounds_long_bodies_by_characters() {
+        let body = "\u{00e9}".repeat(UPSTREAM_ERROR_BODY_MAX_CHARS + 10);
+        let truncated = truncate_error_body(&body);
+        assert!(truncated.ends_with("... (truncated)"));
+        assert_eq!(
+            truncated.chars().count(),
+            UPSTREAM_ERROR_BODY_MAX_CHARS + "... (truncated)".chars().count()
+        );
+    }
+
+    #[test]
+    fn string_headers_rejects_non_string_values() {
+        let headers = Map::from_iter([("x-trace".to_string(), json!(7))]);
+        let err = string_headers("chat completions", Some(headers)).expect_err("non-string value");
+        assert_eq!(
+            err,
+            CoreError::InvalidRequest(
+                "chat completions extra_headers.x-trace must be a string, got number".to_string()
+            )
+        );
+    }
+
+    #[test]
+    fn header_lookup_is_case_insensitive() {
+        let headers = vec![("X-Api-Key".to_string(), "k".to_string())];
+        assert!(has_header(&headers, "x-api-key"));
+        assert!(!has_header(&headers, "authorization"));
+    }
+
+    #[test]
+    fn bearer_detection_requires_a_non_empty_token() {
+        assert!(has_bearer_auth(&[(
+            "Authorization".to_string(),
+            "Bearer abc".to_string()
+        )]));
+        assert!(!has_bearer_auth(&[(
+            "Authorization".to_string(),
+            "Bearer    ".to_string()
+        )]));
+        assert!(!has_bearer_auth(&[(
+            "Authorization".to_string(),
+            "Basic abc".to_string()
+        )]));
+    }
+}

litellm-rust/crates/core/src/lib.rs

@@ -1,8 +1,10 @@
 pub mod audio_transcription;
 pub mod caching;
 pub mod call_lifecycle;
+pub mod chat_completions;
 pub mod constants;
 pub mod error;
+pub mod http_utils;
 pub mod messages;
 pub mod ocr;
 pub mod providers;

litellm-rust/crates/core/src/messages/common_utils.rs

@@ -1,19 +1,15 @@
 use serde_json::{Map, Value};
 
-use crate::constants::MESSAGES_ERROR_BODY_MAX_CHARS;
-use crate::error::{CoreError, CoreResult, json_type_name};
+use crate::error::CoreResult;
+use crate::http_utils::string_headers as shared_string_headers;
 use crate::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG;
 use crate::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG;
 
 use super::transformation::AnthropicMessagesProviderConfig;
 
-pub(super) fn truncate_error_body(body: &str) -> String {
-    if body.chars().count() <= MESSAGES_ERROR_BODY_MAX_CHARS {
-        return body.to_string();
-    }
-    let truncated: String = body.chars().take(MESSAGES_ERROR_BODY_MAX_CHARS).collect();
-    format!("{truncated}... (truncated)")
-}
+pub(super) use crate::http_utils::{has_bearer_auth, has_header, truncate_error_body};
+
+const HEADER_CONTEXT: &str = "messages";
 
 pub(super) fn messages_provider_config(
     provider: &str,
@@ -28,37 +24,5 @@ pub(super) fn messages_provider_config(
 pub(super) fn string_headers(
     extra_headers: Option<Map<String, Value>>,
 ) -> CoreResult<Vec<(String, String)>> {
-    extra_headers
-        .unwrap_or_default()
-        .into_iter()
-        .map(|(key, value)| {
-            value
-                .as_str()
-                .map(|value| (key.clone(), value.to_string()))
-                .ok_or_else(|| {
-                    CoreError::InvalidRequest(format!(
-                        "messages extra_headers.{key} must be a string, got {}",
-                        json_type_name(&value)
-                    ))
-                })
-        })
-        .collect()
-}
-
-pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool {
-    headers
-        .iter()
-        .any(|(key, _)| key.eq_ignore_ascii_case(name))
-}
-
-pub(super) fn has_bearer_auth(headers: &[(String, String)]) -> bool {
-    headers.iter().any(|(name, value)| {
-        if !name.eq_ignore_ascii_case("authorization") {
-            return false;
-        }
-        let value = value.trim();
-        value.len() > 7
-            && value[..7].eq_ignore_ascii_case("bearer ")
-            && !value[7..].trim().is_empty()
-    })
+    shared_string_headers(HEADER_CONTEXT, extra_headers)
 }

litellm-rust/crates/core/src/providers/anthropic/chat_completions/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs

@@ -0,0 +1,444 @@
+use super::*;
+use serde_json::json;
+
+fn messages(value: Value) -> Vec<ChatMessage> {
+    serde_json::from_value(value).expect("valid messages")
+}
+
+fn params(value: Value) -> Map<String, Value> {
+    match value {
+        Value::Object(map) => map,
+        other => panic!("params must be an object, got {other}"),
+    }
+}
+
+fn transform(model: &str, msgs: Value, opts: Value) -> Value {
+    ANTHROPIC_CHAT_COMPLETIONS_CONFIG
+        .transform_request(model, messages(msgs), params(opts))
+        .expect("request transforms")
+        .body
+}
+
+fn transform_response(body: Value) -> CoreResult<ChatCompletionsResponse> {
+    ANTHROPIC_CHAT_COMPLETIONS_CONFIG
+        .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body })
+}
+
+fn reason(msgs: Value, opts: Value) -> Option<Unsupported> {
+    ANTHROPIC_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), &params(opts))
+}
+
+#[test]
+fn builds_the_messages_body_python_builds() {
+    let body = transform(
+        "claude-sonnet-4-5",
+        json!([
+            {"role": "system", "content": "be terse"},
+            {"role": "user", "content": "hi"}
+        ]),
+        json!({"max_tokens": 128, "temperature": 0.2}),
+    );
+    assert_eq!(
+        body,
+        json!({
+            "model": "claude-sonnet-4-5",
+            "messages": [
+                {"role": "user", "content": [{"type": "text", "text": "hi"}]}
+            ],
+            "system": [{"type": "text", "text": "be terse"}],
+            "max_tokens": 128,
+            "temperature": 0.2
+        })
+    );
+}
+
+#[test]
+fn omits_system_when_no_system_message_is_present() {
+    let body = transform(
+        "claude-sonnet-4-5",
+        json!([{"role": "user", "content": "hi"}]),
+        json!({"max_tokens": 16}),
+    );
+    assert!(body.get("system").is_none());
+}
+
+#[test]
+fn merges_consecutive_turns_and_wraps_every_text_in_a_block() {
+    let body = transform(
+        "claude-sonnet-4-5",
+        json!([
+            {"role": "user", "content": "one"},
+            {"role": "user", "content": [{"type": "text", "text": "two"}]},
+            {"role": "assistant", "content": "ack"}
+        ]),
+        json!({"max_tokens": 16}),
+    );
+    assert_eq!(
+        body["messages"],
+        json!([
+            {"role": "user", "content": [
+                {"type": "text", "text": "one"},
+                {"type": "text", "text": "two"}
+            ]},
+            {"role": "assistant", "content": [{"type": "text", "text": "ack"}]}
+        ])
+    );
+}
+
+#[test]
+fn right_strips_a_trailing_assistant_prefill_like_python() {
+    let body = transform(
+        "claude-sonnet-4-5",
+        json!([
+            {"role": "user", "content": "hi"},
+            {"role": "assistant", "content": "Argentina  "}
+        ]),
+        json!({"max_tokens": 16}),
+    );
+    assert_eq!(
+        body["messages"][1]["content"][0]["text"],
+        json!("Argentina")
+    );
+}
+
+#[test]
+fn passes_every_supported_param_through_untouched() {
+    let body = transform(
+        "claude-sonnet-4-5",
+        json!([{"role": "user", "content": "hi"}]),
+        json!({
+            "max_tokens": 64,
+            "temperature": 0.1,
+            "top_p": 0.9,
+            "stop_sequences": ["STOP"]
+        }),
+    );
+    assert_eq!(body["max_tokens"], json!(64));
+    assert_eq!(body["temperature"], json!(0.1));
+    assert_eq!(body["top_p"], json!(0.9));
+    assert_eq!(body["stop_sequences"], json!(["STOP"]));
+}
+
+#[test]
+fn declines_top_k_because_python_gates_it_by_model_below_this_point() {
+    // `temperature` and `top_p` arrive already resolved, because
+    // `map_openai_params` applies `_apply_sampling_param` to them before the
+    // gate runs. `top_k` bypasses that and is gated inside `transform_request`,
+    // the function this route replaces, so forwarding it would send `top_k` to
+    // a model that removed sampling params and take a 400 after the call, where
+    // Python drops it and succeeds.
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"top_k": 40})
+        ),
+        Some(Unsupported("unrecognized request parameter"))
+    );
+}
+
+#[test]
+fn declines_streaming_before_anything_else() {
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"stream": true, "max_tokens": 16})
+        ),
+        Some(Unsupported("streaming"))
+    );
+}
+
+#[test]
+fn accepts_an_explicit_stream_false() {
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"stream": false, "max_tokens": 16})
+        ),
+        None
+    );
+}
+
+#[test]
+fn declines_any_param_outside_the_allowlist() {
+    for param in [
+        json!({"tools": []}),
+        json!({"tool_choice": {"type": "auto"}}),
+        json!({"thinking": {"type": "enabled"}}),
+        json!({"system": "injected"}),
+        json!({"metadata": {"user_id": "u1"}}),
+        json!({"output_config": {"effort": "high"}}),
+    ] {
+        assert_eq!(
+            reason(json!([{"role": "user", "content": "hi"}]), param.clone()),
+            Some(Unsupported("unrecognized request parameter")),
+            "expected {param} to decline"
+        );
+    }
+}
+
+#[test]
+fn declines_tool_calls_tool_results_and_multimodal_content() {
+    assert_eq!(
+        reason(
+            json!([
+                {"role": "user", "content": "hi"},
+                {"role": "assistant", "content": null, "tool_calls": [
+                    {"id": "c1", "type": "function",
+                     "function": {"name": "f", "arguments": "{}"}}
+                ]}
+            ]),
+            json!({})
+        ),
+        Some(Unsupported("unrecognized message field"))
+    );
+    assert_eq!(
+        reason(
+            json!([
+                {"role": "user", "content": "hi"},
+                {"role": "tool", "tool_call_id": "c1", "content": "ok"}
+            ]),
+            json!({})
+        ),
+        Some(Unsupported("unrecognized message field"))
+    );
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": [
+                {"type": "image_url", "image_url": {"url": "https://x/y.png"}}
+            ]}]),
+            json!({})
+        ),
+        Some(Unsupported("non-text message content"))
+    );
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": [
+                {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}
+            ]}]),
+            json!({})
+        ),
+        Some(Unsupported("non-text message content"))
+    );
+}
+
+#[test]
+fn declines_a_message_whose_content_list_is_empty() {
+    // An empty list passes every per-part check, so without this it would reach
+    // the provider as an empty `content` array and fail after the call rather
+    // than declining to Python before it.
+    assert_eq!(
+        reason(json!([{"role": "user", "content": []}]), json!({})),
+        Some(Unsupported("message without content"))
+    );
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]),
+            json!({})
+        ),
+        None
+    );
+}
+
+#[test]
+fn declines_a_conversation_that_does_not_open_on_a_user_turn() {
+    assert_eq!(
+        reason(
+            json!([
+                {"role": "system", "content": "be terse"},
+                {"role": "assistant", "content": "prefill"}
+            ]),
+            json!({})
+        ),
+        Some(Unsupported("conversation does not open on a user turn"))
+    );
+}
+
+#[test]
+fn accepts_a_plain_text_conversation() {
+    assert_eq!(
+        reason(
+            json!([
+                {"role": "system", "content": "be terse"},
+                {"role": "user", "content": "hi"},
+                {"role": "assistant", "content": "hello"},
+                {"role": "user", "content": [{"type": "text", "text": "again"}]}
+            ]),
+            json!({"max_tokens": 16, "temperature": 0.5})
+        ),
+        None
+    );
+}
+
+#[test]
+fn normalizes_a_text_response_into_openai_shape() {
+    let response = transform_response(json!({
+        "id": "msg_123",
+        "type": "message",
+        "role": "assistant",
+        "model": "claude-sonnet-4-5-20260101",
+        "content": [{"type": "text", "text": "hello"}, {"type": "text", "text": " there"}],
+        "stop_reason": "end_turn",
+        "stop_sequence": null,
+        "usage": {"input_tokens": 11, "output_tokens": 4}
+    }))
+    .expect("response transforms");
+
+    assert_eq!(response.model, "claude-sonnet-4-5-20260101");
+    assert_eq!(response.choices.len(), 1);
+    assert_eq!(response.choices[0].index, 0);
+    assert_eq!(response.choices[0].message.role, "assistant");
+    assert_eq!(
+        response.choices[0].message.content.as_deref(),
+        Some("hello there")
+    );
+    assert_eq!(response.choices[0].finish_reason, "stop");
+    assert_eq!(response.usage.prompt_tokens, 11);
+    assert_eq!(response.usage.completion_tokens, 4);
+    assert_eq!(response.usage.total_tokens, 15);
+}
+
+#[test]
+fn folds_cache_tokens_into_prompt_tokens_like_python() {
+    let response = transform_response(json!({
+        "model": "claude-sonnet-4-5",
+        "content": [{"type": "text", "text": "hi"}],
+        "stop_reason": "end_turn",
+        "usage": {
+            "input_tokens": 10,
+            "output_tokens": 2,
+            "cache_read_input_tokens": 5,
+            "cache_creation_input_tokens": 3
+        }
+    }))
+    .expect("response transforms");
+    assert_eq!(response.usage.prompt_tokens, 18);
+    assert_eq!(response.usage.total_tokens, 20);
+    assert_eq!(response.usage.prompt_tokens_details.cached_tokens, 5);
+    assert_eq!(
+        response.usage.prompt_tokens_details.cache_creation_tokens,
+        3
+    );
+    assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10);
+}
+
+#[test]
+fn maps_max_tokens_stop_reason_to_length() {
+    let response = transform_response(json!({
+        "model": "claude-sonnet-4-5",
+        "content": [{"type": "text", "text": "hi"}],
+        "stop_reason": "max_tokens",
+        "usage": {"input_tokens": 1, "output_tokens": 1}
+    }))
+    .expect("response transforms");
+    assert_eq!(response.choices[0].finish_reason, "length");
+}
+
+#[test]
+fn a_refusal_returns_the_completion_python_returns() {
+    // `refusal` is a stop_reason, not a content block type, so the content is
+    // ordinary text and this normalizes rather than declining. Python maps it
+    // to content_filter in _FINISH_REASON_MAP and returns the completion.
+    let response = transform_response(json!({
+        "model": "claude-sonnet-4-5",
+        "content": [{"type": "text", "text": "I can't help with that."}],
+        "stop_reason": "refusal",
+        "usage": {"input_tokens": 9, "output_tokens": 6}
+    }))
+    .expect("a refusal still transforms");
+    assert_eq!(response.choices[0].finish_reason, "content_filter");
+    assert_eq!(
+        response.choices[0].message.content.as_deref(),
+        Some("I can't help with that.")
+    );
+}
+
+#[test]
+fn reports_no_content_rather_than_an_empty_string() {
+    let response = transform_response(json!({
+        "model": "claude-sonnet-4-5",
+        "content": [],
+        "stop_reason": "end_turn",
+        "usage": {"input_tokens": 1, "output_tokens": 0}
+    }))
+    .expect("response transforms");
+    assert_eq!(response.choices[0].message.content, None);
+}
+
+#[test]
+fn response_carries_no_id_so_python_keeps_its_chatcmpl_id() {
+    let response = transform_response(json!({
+        "id": "msg_should_not_leak",
+        "model": "claude-sonnet-4-5",
+        "content": [{"type": "text", "text": "hi"}],
+        "stop_reason": "end_turn",
+        "usage": {"input_tokens": 1, "output_tokens": 1}
+    }))
+    .expect("response transforms");
+    let value = serde_json::to_value(response).expect("serializable");
+    assert!(
+        value.get("id").is_none(),
+        "the rust response must not carry an id, got {value}"
+    );
+}
+
+#[test]
+fn declines_a_response_carrying_a_non_text_block() {
+    let err = transform_response(json!({
+        "model": "claude-sonnet-4-5",
+        "content": [{"type": "tool_use", "id": "t1", "name": "f", "input": {}}],
+        "stop_reason": "tool_use",
+        "usage": {"input_tokens": 1, "output_tokens": 1}
+    }))
+    .expect_err("non-text block");
+    assert_eq!(
+        err,
+        CoreError::Unsupported("non-text response content block")
+    );
+}
+
+#[test]
+fn errors_on_a_response_missing_required_fields() {
+    assert_eq!(
+        transform_response(json!("nope")).expect_err("not an object"),
+        CoreError::InvalidResponse("messages response is not an object".to_string())
+    );
+    assert_eq!(
+        transform_response(json!({"model": "m", "usage": {}})).expect_err("no content"),
+        CoreError::MissingField("content")
+    );
+    assert_eq!(
+        transform_response(json!({"model": "m", "content": []})).expect_err("no usage"),
+        CoreError::MissingField("usage")
+    );
+    assert_eq!(
+        transform_response(json!({"content": [], "usage": {}})).expect_err("no model"),
+        CoreError::MissingField("model")
+    );
+}
+
+#[test]
+fn resolves_the_messages_url_and_x_api_key_auth() {
+    let config = &ANTHROPIC_CHAT_COMPLETIONS_CONFIG;
+    assert_eq!(
+        config
+            .complete_url(None, "claude-sonnet-4-5", &Map::new(), &|_| None)
+            .expect("url builds"),
+        "https://api.anthropic.com/v1/messages"
+    );
+    assert_eq!(
+        config
+            .auth(Some("sk-x"), "claude-sonnet-4-5", &Map::new(), &|_| None)
+            .expect("auth resolves"),
+        ChatCompletionsAuth::Header {
+            name: "x-api-key",
+            value: "sk-x".to_string()
+        }
+    );
+    assert_eq!(
+        config.default_headers(),
+        &[
+            ("anthropic-version", "2023-06-01"),
+            ("content-type", "application/json"),
+        ]
+    );
+}

litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs

@@ -0,0 +1,211 @@
+use serde_json::{Map, Value, json};
+
+use crate::chat_completions::conversation::{Conversation, build_conversation};
+use crate::chat_completions::transformation::{
+    ChatCompletionsAuth, ChatCompletionsProviderConfig, Unsupported, unsupported_message,
+    unsupported_param,
+};
+use crate::chat_completions::types::{
+    ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage,
+    ProviderChatRequestData, ProviderChatResponseData,
+};
+use crate::constants::ANTHROPIC_OAUTH_TOKEN_PREFIX;
+use crate::error::{CoreError, CoreResult};
+use crate::providers::anthropic::messages::transformation::{
+    complete_anthropic_url, resolve_anthropic_api_key,
+};
+
+use crate::chat_completions::response_utils::{finish_reason_for, unix_now, usage_from_parts};
+
+/// Anthropic parameter names, post `map_openai_params`, that the Rust path can
+/// place verbatim in the Messages body.
+///
+/// `top_k` is deliberately absent even though the Messages API takes it.
+/// `temperature` and `top_p` reach this gate already resolved, because
+/// `map_openai_params` runs first and applies `_apply_sampling_param` to them.
+/// `top_k` bypasses `map_openai_params` entirely, so Python applies that same
+/// per-model gate inside `transform_request`, the function this route replaces.
+/// Forwarding it would send `top_k` to a model that removed sampling params and
+/// take a 400 after the call, where Python drops it and succeeds.
+const SUPPORTED_PARAMS: &[&str] = &["max_tokens", "temperature", "top_p", "stop_sequences"];
+
+pub struct AnthropicChatCompletionsConfig;
+
+pub const ANTHROPIC_CHAT_COMPLETIONS_CONFIG: AnthropicChatCompletionsConfig =
+    AnthropicChatCompletionsConfig;
+
+fn text_block(text: &str) -> Value {
+    json!({"type": "text", "text": text})
+}
+
+fn anthropic_body(model: &str, conversation: &Conversation, params: Map<String, Value>) -> Value {
+    let messages: Vec<Value> = conversation
+        .turns
+        .iter()
+        .map(|turn| {
+            json!({
+                "role": turn.role.as_str(),
+                "content": turn.texts.iter().map(|text| text_block(text)).collect::<Vec<_>>(),
+            })
+        })
+        .collect();
+
+    let system: Vec<Value> = conversation.system.iter().map(|s| text_block(s)).collect();
+
+    let body = Map::from_iter(
+        [
+            ("model".to_string(), json!(model)),
+            ("messages".to_string(), json!(messages)),
+        ]
+        .into_iter()
+        // Python builds `{"model", "messages", **optional_params}` with
+        // `system` already folded into optional_params, so a caller-supplied
+        // key of the same name wins here too.
+        .chain((!system.is_empty()).then(|| ("system".to_string(), json!(system))))
+        .chain(params),
+    );
+    Value::Object(body)
+}
+
+impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig {
+    fn complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        _optional_params: &Map<String, Value>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<String> {
+        Ok(complete_anthropic_url(api_base, env_lookup))
+    }
+
+    fn auth(
+        &self,
+        api_key: Option<&str>,
+        _model: &str,
+        _optional_params: &Map<String, Value>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<ChatCompletionsAuth> {
+        Ok(ChatCompletionsAuth::Header {
+            name: "x-api-key",
+            value: resolve_anthropic_api_key(api_key, env_lookup)?,
+        })
+    }
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        &[
+            ("anthropic-version", "2023-06-01"),
+            ("content-type", "application/json"),
+        ]
+    }
+
+    /// An OAuth bearer is the whole credential: Python's `validate_environment`
+    /// authenticates with it and drops `x-api-key` rather than resolving one, so
+    /// the resolved key must not be applied over the top. Any other forwarded
+    /// `authorization` is unrelated to this header and does not defer, which is
+    /// also what Python does: it sends the deployment's `x-api-key` alongside.
+    fn defers_to_forwarded_auth(&self, headers: &[(String, String)]) -> bool {
+        headers.iter().any(|(name, value)| {
+            name.eq_ignore_ascii_case("authorization")
+                && value
+                    .strip_prefix("Bearer ")
+                    .is_some_and(|token| token.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX))
+        })
+    }
+
+    fn supported_params(&self) -> &'static [&'static str] {
+        SUPPORTED_PARAMS
+    }
+
+    fn unsupported_reason(
+        &self,
+        messages: &[ChatMessage],
+        optional_params: &Map<String, Value>,
+    ) -> Option<Unsupported> {
+        unsupported_param(SUPPORTED_PARAMS, &[], optional_params)
+            .or_else(|| messages.iter().find_map(unsupported_message))
+            // Anthropic rejects a request whose first turn is not a user turn.
+            // Python only repairs that under `litellm.modify_params`, which the
+            // core cannot observe, so decline instead of guessing.
+            .or_else(|| {
+                (!build_conversation(messages).opens_on_user_turn())
+                    .then_some(Unsupported("conversation does not open on a user turn"))
+            })
+    }
+
+    fn transform_request(
+        &self,
+        model: &str,
+        messages: Vec<ChatMessage>,
+        optional_params: Map<String, Value>,
+    ) -> CoreResult<ProviderChatRequestData> {
+        Ok(ProviderChatRequestData {
+            body: anthropic_body(model, &build_conversation(&messages), optional_params),
+        })
+    }
+
+    fn transform_response(
+        &self,
+        _model: &str,
+        response: ProviderChatResponseData,
+    ) -> CoreResult<ChatCompletionsResponse> {
+        let body = response.body.as_object().ok_or_else(|| {
+            CoreError::InvalidResponse("messages response is not an object".into())
+        })?;
+
+        let content = body
+            .get("content")
+            .and_then(Value::as_array)
+            .ok_or(CoreError::MissingField("content"))?;
+        // The route declines tool and thinking requests, so a non-text block
+        // means the response carries something this path never asked for.
+        // Decline rather than silently dropping it; the host falls back.
+        if content
+            .iter()
+            .any(|block| block.get("type").and_then(Value::as_str) != Some("text"))
+        {
+            return Err(CoreError::Unsupported("non-text response content block"));
+        }
+        let text: String = content
+            .iter()
+            .filter_map(|block| block.get("text").and_then(Value::as_str))
+            .collect();
+
+        let usage = body
+            .get("usage")
+            .and_then(Value::as_object)
+            .ok_or(CoreError::MissingField("usage"))?;
+        let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0);
+
+        Ok(ChatCompletionsResponse {
+            created: unix_now(),
+            model: body
+                .get("model")
+                .and_then(Value::as_str)
+                .ok_or(CoreError::MissingField("model"))?
+                .to_string(),
+            choices: vec![ChatCompletionsChoice {
+                index: 0,
+                message: ChatCompletionsChoiceMessage {
+                    role: "assistant".to_string(),
+                    content: (!text.is_empty()).then_some(text),
+                },
+                finish_reason: finish_reason_for(
+                    body.get("stop_reason")
+                        .and_then(Value::as_str)
+                        .unwrap_or(""),
+                )
+                .to_string(),
+            }],
+            usage: usage_from_parts(
+                field("input_tokens"),
+                field("output_tokens"),
+                field("cache_read_input_tokens"),
+                field("cache_creation_input_tokens"),
+            ),
+        })
+    }
+}
+
+#[cfg(test)]
+#[path = "tests.rs"]
+mod tests;

litellm-rust/crates/core/src/providers/anthropic/mod.rs

@@ -1 +1,2 @@
+pub mod chat_completions;
 pub mod messages;

litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs

@@ -8,11 +8,8 @@ use crate::audio_transcription::types::{
 };
 use crate::error::{CoreError, CoreResult, json_type_name};
 
-use super::aws_base::AwsAuthConfig;
-use super::constants::{
-    AWS_REGION, AWS_REGION_NAME, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE,
-    DEFAULT_BEDROCK_REGION,
-};
+pub use super::aws_base::{aws_auth_config, bedrock_model_id_and_region, resolve_bedrock_region};
+use super::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE};
 
 const SUPPORTED_PARAMS: &[&str] = &["language", "prompt", "temperature", "response_format"];
 
@@ -21,64 +18,6 @@ pub static BEDROCK_AUDIO_TRANSCRIPTION_CONFIG: BedrockAudioTranscriptionConfig =
 
 pub struct BedrockAudioTranscriptionConfig;
 
-pub fn bedrock_model_id_and_region(model: &str) -> (String, Option<String>) {
-    let mut stripped = model;
-    for prefix in ["bedrock/converse/", "bedrock/", "converse/"] {
-        if let Some(value) = stripped.strip_prefix(prefix) {
-            stripped = value;
-            break;
-        }
-    }
-    let mut region = None;
-    if let Some((candidate, remainder)) = stripped.split_once('/')
-        && is_bedrock_region(candidate)
-    {
-        region = Some(candidate.to_string());
-        stripped = remainder;
-    }
-    for prefix in ["nova-2/", "nova/"] {
-        if let Some(value) = stripped.strip_prefix(prefix) {
-            stripped = value;
-            break;
-        }
-    }
-    if region.is_none() {
-        region = stripped
-            .strip_prefix("arn:")
-            .and_then(|value| value.split(':').nth(3))
-            .filter(|value| !value.is_empty())
-            .map(str::to_string);
-    }
-    (stripped.to_string(), region)
-}
-
-fn is_bedrock_region(value: &str) -> bool {
-    value.len() > 3
-        && value.contains('-')
-        && value
-            .chars()
-            .all(|char| char.is_ascii_alphanumeric() || char == '-')
-}
-
-pub fn resolve_bedrock_region(
-    model_region: Option<&str>,
-    optional_params: &Map<String, Value>,
-    env_lookup: &dyn Fn(&str) -> Option<String>,
-) -> String {
-    if let Some(region) = optional_params
-        .get("aws_region_name")
-        .and_then(Value::as_str)
-    {
-        return region.to_string();
-    }
-    if let Some(region) = model_region {
-        return region.to_string();
-    }
-    env_lookup(AWS_REGION_NAME)
-        .or_else(|| env_lookup(AWS_REGION))
-        .unwrap_or_else(|| DEFAULT_BEDROCK_REGION.to_string())
-}
-
 fn audio_fields(audio: Value) -> CoreResult<(String, String)> {
     let object = audio.as_object().ok_or_else(|| CoreError::InvalidType {
         expected: "object",
@@ -203,32 +142,6 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig {
     }
 }
 
-pub fn aws_auth_config(
-    optional_params: &Map<String, Value>,
-    env_lookup: &dyn Fn(&str) -> Option<String>,
-) -> AwsAuthConfig {
-    let value = |key: &str| {
-        optional_params
-            .get(key)
-            .and_then(Value::as_str)
-            .map(str::to_string)
-    };
-    let env = |key: &str| env_lookup(key);
-    AwsAuthConfig {
-        access_key_id: value("aws_access_key_id").or_else(|| env("AWS_ACCESS_KEY_ID")),
-        secret_access_key: value("aws_secret_access_key").or_else(|| env("AWS_SECRET_ACCESS_KEY")),
-        session_token: value("aws_session_token").or_else(|| env("AWS_SESSION_TOKEN")),
-        region_name: value("aws_region_name").or_else(|| env(AWS_REGION_NAME)),
-        session_name: value("aws_session_name").or_else(|| env("AWS_SESSION_NAME")),
-        profile_name: value("aws_profile_name").or_else(|| env("AWS_PROFILE_NAME")),
-        role_name: value("aws_role_name").or_else(|| env("AWS_ROLE_NAME")),
-        web_identity_token: value("aws_web_identity_token")
-            .or_else(|| env("AWS_WEB_IDENTITY_TOKEN")),
-        sts_endpoint: value("aws_sts_endpoint").or_else(|| env("AWS_STS_ENDPOINT")),
-        external_id: value("aws_external_id").or_else(|| env("AWS_EXTERNAL_ID")),
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

litellm-rust/crates/core/src/providers/bedrock/aws_base.rs

@@ -12,13 +12,15 @@ use aws_sigv4::http_request::{
 };
 use aws_sigv4::sign::v4;
 use aws_smithy_runtime_api::client::identity::Identity;
+use serde_json::{Map, Value};
 use sha2::{Digest, Sha256};
 
 use super::constants::{
-    AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_ROLE_ARN,
-    AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN, AWS_STS_ENDPOINT,
-    AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE, BEDROCK_SERVICE,
-    DEFAULT_SESSION_NAME_PREFIX,
+    AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION, AWS_REGION_NAME,
+    AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN,
+    AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE,
+    BEDROCK_SERVICE, DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX,
+    SIGV4_COMPUTED_HEADER_NAMES,
 };
 
 const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60);
@@ -401,6 +403,33 @@ fn default_session_name() -> String {
     format!("{DEFAULT_SESSION_NAME_PREFIX}-{seconds}")
 }
 
+/// The subset of `headers` SigV4 should cover.
+///
+/// Python signs only these and reattaches the rest afterwards, so a forwarded
+/// client header cannot change the canonical request and invalidate the
+/// signature. Signing everything instead makes the request 403 on a header the
+/// caller supplied, on a deployment that works on the Python path.
+pub fn aws_signature_headers(headers: &BTreeMap<String, String>) -> BTreeMap<String, String> {
+    headers
+        .iter()
+        .filter(|(name, _)| {
+            let name = name.to_ascii_lowercase();
+            AWS_SIGNED_HEADER_NAMES.contains(&name.as_str())
+                || name.starts_with("x-amz-")
+                || name.starts_with("x-amzn-")
+        })
+        .map(|(name, value)| (name.clone(), value.clone()))
+        .collect()
+}
+
+/// Whether the signer produces `name` itself.
+///
+/// Python's reattach loop skips these, so a caller-supplied copy never reaches
+/// the wire next to the computed one.
+pub fn is_sigv4_computed_header(name: &str) -> bool {
+    SIGV4_COMPUTED_HEADER_NAMES.contains(&name.to_ascii_lowercase().as_str())
+}
+
 pub fn sign_bedrock_post(
     url: &str,
     body: &[u8],
@@ -441,6 +470,121 @@ pub fn sign_bedrock_post(
         .collect())
 }
 
+/// Model-id and region parsing shared by every Bedrock route.
+pub fn bedrock_model_id_and_region(model: &str) -> (String, Option<String>) {
+    let mut stripped = model;
+    for prefix in ["bedrock/converse/", "bedrock/", "converse/"] {
+        if let Some(value) = stripped.strip_prefix(prefix) {
+            stripped = value;
+            break;
+        }
+    }
+    let mut region = None;
+    if let Some((candidate, remainder)) = stripped.split_once('/')
+        && is_bedrock_region(candidate)
+    {
+        region = Some(candidate.to_string());
+        stripped = remainder;
+    }
+    for prefix in ["nova-2/", "nova/"] {
+        if let Some(value) = stripped.strip_prefix(prefix) {
+            stripped = value;
+            break;
+        }
+    }
+    if region.is_none() {
+        // Python splits the whole ARN and takes field 3, the region. Stripping
+        // `arn:` first shifts every field down one, so the region is field 2
+        // here; field 3 is the account id.
+        region = stripped
+            .strip_prefix("arn:")
+            .and_then(|value| value.split(':').nth(2))
+            .filter(|value| !value.is_empty())
+            .map(str::to_string);
+    }
+    (stripped.to_string(), region)
+}
+
+fn is_bedrock_region(value: &str) -> bool {
+    value.len() > 3
+        && value.contains('-')
+        && value
+            .chars()
+            .all(|char| char.is_ascii_alphanumeric() || char == '-')
+}
+
+pub fn resolve_bedrock_region(
+    model_region: Option<&str>,
+    optional_params: &Map<String, Value>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> String {
+    if let Some(region) = optional_params
+        .get("aws_region_name")
+        .and_then(Value::as_str)
+    {
+        return region.to_string();
+    }
+    if let Some(region) = model_region {
+        return region.to_string();
+    }
+    env_lookup(AWS_REGION_NAME)
+        .or_else(|| env_lookup(AWS_REGION))
+        .unwrap_or_else(|| DEFAULT_BEDROCK_REGION.to_string())
+}
+
+pub fn aws_auth_config(
+    optional_params: &Map<String, Value>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> AwsAuthConfig {
+    let value = |key: &str| {
+        optional_params
+            .get(key)
+            .and_then(Value::as_str)
+            .map(str::to_string)
+    };
+    let env = |key: &str| env_lookup(key);
+    AwsAuthConfig {
+        access_key_id: value("aws_access_key_id").or_else(|| env("AWS_ACCESS_KEY_ID")),
+        secret_access_key: value("aws_secret_access_key").or_else(|| env("AWS_SECRET_ACCESS_KEY")),
+        session_token: value("aws_session_token").or_else(|| env("AWS_SESSION_TOKEN")),
+        region_name: value("aws_region_name").or_else(|| env(AWS_REGION_NAME)),
+        session_name: value("aws_session_name").or_else(|| env("AWS_SESSION_NAME")),
+        profile_name: value("aws_profile_name").or_else(|| env("AWS_PROFILE_NAME")),
+        role_name: value("aws_role_name").or_else(|| env("AWS_ROLE_NAME")),
+        web_identity_token: value("aws_web_identity_token")
+            .or_else(|| env("AWS_WEB_IDENTITY_TOKEN")),
+        sts_endpoint: value("aws_sts_endpoint").or_else(|| env("AWS_STS_ENDPOINT")),
+        external_id: value("aws_external_id").or_else(|| env("AWS_EXTERNAL_ID")),
+    }
+}
+
+/// Credentials a host resolved through its own chain and handed down verbatim.
+///
+/// A host with its own resolution (LiteLLM's Python `BaseAWSLLM`, which reads
+/// profiles, STS and boto sessions) passes the result here so the core signs
+/// with exactly those. Without this the core would re-derive from ambient
+/// state, where an unrelated `AWS_ROLE_NAME` or `AWS_PROFILE_NAME` in the
+/// environment outranks explicit keys in [`classify_auth`] and the two sides
+/// would sign as different principals.
+pub fn host_supplied_credentials(optional_params: &Map<String, Value>) -> Option<Credentials> {
+    let value = |key: &str| {
+        optional_params
+            .get(key)
+            .and_then(Value::as_str)
+            .map(str::trim)
+            .filter(|value| !value.is_empty())
+    };
+    let access_key_id = value("aws_access_key_id")?;
+    let secret_access_key = value("aws_secret_access_key")?;
+    Some(Credentials::new(
+        access_key_id,
+        secret_access_key,
+        value("aws_session_token").map(str::to_string),
+        None,
+        "litellm-host-supplied",
+    ))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -458,6 +602,18 @@ mod tests {
         )
     }
 
+    #[test]
+    fn reads_the_region_field_of_a_model_arn_not_the_account_id() {
+        // Python's `_get_aws_region_from_model_arn` splits the whole ARN and
+        // takes field 3. Stripping `arn:` first shifts every field down one, so
+        // the region is field 2 here. Taking field 3 after the strip returns
+        // the account id, which is not a region at all.
+        let (_, region) = bedrock_model_id_and_region(
+            "bedrock/arn:aws:bedrock:us-west-2:123456789012:foundation-model/anthropic.claude-v2",
+        );
+        assert_eq!(region.as_deref(), Some("us-west-2"));
+    }
+
     #[test]
     fn classification_preserves_python_precedence() {
         let config = AwsAuthConfig {
@@ -610,6 +766,52 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn a_forwarded_client_header_is_not_folded_into_the_signature() {
+        // Python signs only the AWS header set, so a header a caller forwarded
+        // cannot change the canonical request. Signing it instead makes the
+        // request 403 the moment anything on the wire rewrites or drops it.
+        let (url, body, mut headers) = parity_inputs();
+        headers.insert("x-request-id".to_string(), "abc-123".to_string());
+        headers.insert("Accept-Encoding".to_string(), "gzip".to_string());
+        headers.insert("x-amzn-trace-id".to_string(), "Root=1-abc".to_string());
+        let signable = aws_signature_headers(&headers);
+
+        assert!(!signable.contains_key("x-request-id"));
+        assert!(!signable.contains_key("Accept-Encoding"));
+        // The AWS-prefixed one is genuinely part of the signature.
+        assert!(signable.contains_key("x-amzn-trace-id"));
+        assert!(signable.contains_key("Content-Type"));
+
+        let credentials = Credentials::new(
+            "AKIDEXAMPLE",
+            "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY",
+            None,
+            None,
+            "test",
+        );
+        let signed = sign_bedrock_post(
+            &url,
+            &body,
+            &signable,
+            "us-east-1",
+            &credentials,
+            SystemTime::UNIX_EPOCH,
+        )
+        .expect("signs");
+        let authorization = signed
+            .get("Authorization")
+            .expect("carries an authorization header");
+        assert!(
+            !authorization.contains("x-request-id"),
+            "forwarded header reached SignedHeaders: {authorization}"
+        );
+        assert!(
+            !authorization.contains("accept-encoding"),
+            "forwarded header reached SignedHeaders: {authorization}"
+        );
+    }
+
     #[test]
     fn signing_matches_botocore_golden_vector() {
         let (url, body, headers) = parity_inputs();

litellm-rust/crates/core/src/providers/bedrock/chat_completions/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs

@@ -0,0 +1,580 @@
+use super::*;
+use serde_json::json;
+
+fn messages(value: Value) -> Vec<ChatMessage> {
+    serde_json::from_value(value).expect("valid messages")
+}
+
+fn params(value: Value) -> Map<String, Value> {
+    match value {
+        Value::Object(map) => map,
+        other => panic!("params must be an object, got {other}"),
+    }
+}
+
+fn transform(msgs: Value, opts: Value) -> Value {
+    BEDROCK_CHAT_COMPLETIONS_CONFIG
+        .transform_request(
+            "anthropic.claude-sonnet-4-5-v1:0",
+            messages(msgs),
+            params(opts),
+        )
+        .expect("request transforms")
+        .body
+}
+
+fn transform_response(body: Value) -> CoreResult<ChatCompletionsResponse> {
+    BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response(
+        "anthropic.claude-sonnet-4-5-v1:0",
+        ProviderChatResponseData { body },
+    )
+}
+
+fn reason(msgs: Value, opts: Value) -> Option<Unsupported> {
+    BEDROCK_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), &params(opts))
+}
+
+#[test]
+fn builds_the_converse_body_python_builds() {
+    let body = transform(
+        json!([
+            {"role": "system", "content": "be terse"},
+            {"role": "user", "content": "hi"}
+        ]),
+        json!({"maxTokens": 128, "temperature": 0.2}),
+    );
+    assert_eq!(
+        body,
+        json!({
+            "inferenceConfig": {"maxTokens": 128, "temperature": 0.2},
+            "messages": [{"role": "user", "content": [{"text": "hi"}]}],
+            "system": [{"text": "be terse"}]
+        })
+    );
+}
+
+#[test]
+fn always_emits_inference_config_even_when_empty() {
+    let body = transform(json!([{"role": "user", "content": "hi"}]), json!({}));
+    assert_eq!(body["inferenceConfig"], json!({}));
+    assert!(body.get("system").is_none());
+}
+
+#[test]
+fn places_only_inference_params_in_inference_config() {
+    let body = transform(
+        json!([{"role": "user", "content": "hi"}]),
+        json!({
+            "maxTokens": 64,
+            "temperature": 0.1,
+            "topP": 0.9,
+            "stopSequences": ["STOP"]
+        }),
+    );
+    assert_eq!(
+        body["inferenceConfig"],
+        json!({"maxTokens": 64, "temperature": 0.1, "topP": 0.9, "stopSequences": ["STOP"]})
+    );
+    assert!(body.get("additionalModelRequestFields").is_none());
+}
+
+#[test]
+fn merges_consecutive_user_turns_into_one_message() {
+    let body = transform(
+        json!([
+            {"role": "user", "content": "one"},
+            {"role": "user", "content": [{"type": "text", "text": "two"}]},
+            {"role": "assistant", "content": "ack"},
+            {"role": "user", "content": "three"}
+        ]),
+        json!({}),
+    );
+    assert_eq!(
+        body["messages"],
+        json!([
+            {"role": "user", "content": [{"text": "one"}, {"text": "two"}]},
+            {"role": "assistant", "content": [{"text": "ack"}]},
+            {"role": "user", "content": [{"text": "three"}]}
+        ])
+    );
+}
+
+#[test]
+fn declines_streaming() {
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"stream": true})
+        ),
+        Some(Unsupported("streaming"))
+    );
+}
+
+#[test]
+fn declines_top_k_because_python_routes_it_by_base_model() {
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"topK": 40})
+        ),
+        Some(Unsupported("unrecognized request parameter"))
+    );
+}
+
+#[test]
+fn declines_tools_and_other_params_outside_the_allowlist() {
+    for param in [
+        json!({"tools": []}),
+        json!({"tool_choice": {"auto": {}}}),
+        json!({"thinking": {"type": "enabled"}}),
+        json!({"requestMetadata": {"k": "v"}}),
+        json!({"outputConfig": {}}),
+        json!({"_parallel_tool_use_config": {}}),
+    ] {
+        assert_eq!(
+            reason(json!([{"role": "user", "content": "hi"}]), param.clone()),
+            Some(Unsupported("unrecognized request parameter")),
+            "expected {param} to decline"
+        );
+    }
+}
+
+#[test]
+fn declines_blank_text_rather_than_substituting_the_anthropic_placeholder() {
+    for content in [
+        json!(""),
+        json!("   "),
+        json!([{"type": "text", "text": " "}]),
+    ] {
+        assert_eq!(
+            reason(
+                json!([{"role": "user", "content": content}, {"role": "user", "content": "hi"}]),
+                json!({})
+            ),
+            Some(Unsupported("blank message text")),
+            "expected blank content {content} to decline"
+        );
+    }
+}
+
+#[test]
+fn declines_a_message_whose_content_list_is_empty() {
+    // The blank-text check scans parts, so an empty list clears it; Converse
+    // rejects an empty `content` array, which is a decline the core owes the
+    // host before the call rather than an error after it.
+    assert_eq!(
+        reason(json!([{"role": "user", "content": []}]), json!({})),
+        Some(Unsupported("message without content"))
+    );
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]),
+            json!({})
+        ),
+        None
+    );
+}
+
+#[test]
+fn declines_a_conversation_that_opens_or_closes_on_an_assistant_turn() {
+    assert_eq!(
+        reason(
+            json!([
+                {"role": "assistant", "content": "prefill"},
+                {"role": "user", "content": "hi"}
+            ]),
+            json!({})
+        ),
+        Some(Unsupported(
+            "conversation does not run user turn to user turn"
+        ))
+    );
+    assert_eq!(
+        reason(
+            json!([
+                {"role": "user", "content": "hi"},
+                {"role": "assistant", "content": "prefill"}
+            ]),
+            json!({})
+        ),
+        Some(Unsupported(
+            "conversation does not run user turn to user turn"
+        ))
+    );
+}
+
+#[test]
+fn accepts_a_user_to_user_text_conversation() {
+    assert_eq!(
+        reason(
+            json!([
+                {"role": "system", "content": "be terse"},
+                {"role": "user", "content": "hi"},
+                {"role": "assistant", "content": "hello"},
+                {"role": "user", "content": "again"}
+            ]),
+            json!({"maxTokens": 16})
+        ),
+        None
+    );
+}
+
+#[test]
+fn builds_the_converse_url_from_the_region_in_the_model_id() {
+    let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG;
+    assert_eq!(
+        config
+            .complete_url(None, "us-east-1/anthropic.claude-v2", &Map::new(), &|_| {
+                None
+            })
+            .expect("url builds"),
+        "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
+    );
+}
+
+#[test]
+fn falls_back_to_the_region_env_then_the_default_region() {
+    let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG;
+    let with_env = |key: &str| (key == "AWS_REGION_NAME").then(|| "eu-west-1".to_string());
+    assert_eq!(
+        config
+            .complete_url(None, "anthropic.claude-v2", &Map::new(), &with_env)
+            .expect("url builds"),
+        "https://bedrock-runtime.eu-west-1.amazonaws.com/model/anthropic.claude-v2/converse"
+    );
+    assert_eq!(
+        config
+            .complete_url(None, "anthropic.claude-v2", &Map::new(), &|_| None)
+            .expect("url builds"),
+        "https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-v2/converse"
+    );
+}
+
+#[test]
+fn prefers_an_explicit_runtime_endpoint_over_the_api_base() {
+    let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG;
+    let overrides = params(json!({"aws_bedrock_runtime_endpoint": "https://vpce.internal/"}));
+    assert_eq!(
+        config
+            .complete_url(
+                Some("https://ignored.example"),
+                "anthropic.claude-v2",
+                &overrides,
+                &|_| None
+            )
+            .expect("url builds"),
+        "https://vpce.internal/model/anthropic.claude-v2/converse"
+    );
+}
+
+#[test]
+fn signs_with_sigv4_in_the_resolved_region() {
+    let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG;
+    assert_eq!(
+        config
+            .auth(
+                None,
+                "eu-central-1/anthropic.claude-v2",
+                &Map::new(),
+                &|_| None
+            )
+            .expect("auth resolves"),
+        ChatCompletionsAuth::AwsSigV4 {
+            region: "eu-central-1".to_string()
+        }
+    );
+}
+
+#[test]
+fn a_bearer_token_outranks_sigv4_the_way_python_resolves_it() {
+    // Python's get_request_headers reads `api_key` as the Bedrock bearer token
+    // and only falls back to the env when the caller passed none, so each case
+    // pins one of its precedence rules. Signing as the host principal when a
+    // bearer identity is configured would cross an account and quota boundary.
+    let bedrock_env =
+        |key: &str| (key == "AWS_BEARER_TOKEN_BEDROCK").then(|| "from-env".to_string());
+    let no_env = |_: &str| None;
+    let resolve = |api_key, env: &dyn Fn(&str) -> Option<String>| {
+        BEDROCK_CHAT_COMPLETIONS_CONFIG
+            .auth(
+                api_key,
+                "eu-central-1/anthropic.claude-v2",
+                &Map::new(),
+                env,
+            )
+            .expect("auth resolves")
+    };
+    let bearer = |token: &str| ChatCompletionsAuth::Bearer {
+        token: token.to_string(),
+    };
+    let sigv4 = ChatCompletionsAuth::AwsSigV4 {
+        region: "eu-central-1".to_string(),
+    };
+
+    // A caller-supplied key is the bearer token, and outranks the env.
+    assert_eq!(
+        resolve(Some("bedrock-api-key"), &bedrock_env),
+        bearer("bedrock-api-key")
+    );
+    // No key, so the env supplies it.
+    assert_eq!(resolve(None, &bedrock_env), bearer("from-env"));
+    // An empty key is not a bearer token, and deliberately does NOT reach for
+    // the env, which is what Python's `is not None` check does.
+    assert_eq!(resolve(Some(""), &bedrock_env), sigv4);
+    // Whitespace is truthy in Python, so it stays a bearer token rather than
+    // silently becoming a host-credentialed SigV4 request.
+    assert_eq!(resolve(Some("  "), &no_env), bearer("  "));
+    // Neither present, so SigV4 as before.
+    assert_eq!(resolve(None, &no_env), sigv4);
+}
+
+#[test]
+fn normalizes_a_converse_response_into_openai_shape() {
+    let response = transform_response(json!({
+        "output": {"message": {"role": "assistant", "content": [
+            {"text": "hello"}, {"text": " there"}
+        ]}},
+        "stopReason": "end_turn",
+        "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}
+    }))
+    .expect("response transforms");
+
+    assert_eq!(response.model, "anthropic.claude-sonnet-4-5-v1:0");
+    assert_eq!(
+        response.choices[0].message.content.as_deref(),
+        Some("hello there")
+    );
+    assert_eq!(response.choices[0].finish_reason, "stop");
+    assert_eq!(response.usage.prompt_tokens, 11);
+    assert_eq!(response.usage.completion_tokens, 4);
+    assert_eq!(response.usage.total_tokens, 15);
+}
+
+#[test]
+fn maps_converse_stop_reasons_python_maps() {
+    for (provider_reason, expected) in [
+        ("end_turn", "stop"),
+        ("stop_sequence", "stop"),
+        ("max_tokens", "length"),
+        ("guardrail_intervened", "content_filter"),
+        // Converse emits this one, and Python's `_FINISH_REASON_MAP` carries
+        // it. Folding it into `stop` reports a filtered completion as a normal
+        // one to anything keying on the finish reason.
+        ("content_filtered", "content_filter"),
+        ("content_filter", "content_filter"),
+    ] {
+        let response = transform_response(json!({
+            "output": {"message": {"content": [{"text": "x"}]}},
+            "stopReason": provider_reason,
+            "usage": {"inputTokens": 1, "outputTokens": 1}
+        }))
+        .expect("response transforms");
+        assert_eq!(
+            response.choices[0].finish_reason, expected,
+            "stopReason {provider_reason}"
+        );
+    }
+}
+
+#[test]
+fn reports_an_empty_converse_answer_as_an_empty_string_not_null() {
+    // Converse assigns the joined text unconditionally
+    // (`chat_completion_message["content"] = content_str`), unlike Anthropic's
+    // `merged_text or None`, so an empty answer is `""` on both paths. A caller
+    // calling `.strip()` on it would break on the Rust path alone. Reachable
+    // through a filtered or guardrail-intervened response.
+    for content in [json!([]), json!([{"text": ""}])] {
+        let response = transform_response(json!({
+            "output": {"message": {"content": content}},
+            "stopReason": "content_filtered",
+            "usage": {"inputTokens": 1, "outputTokens": 0}
+        }))
+        .expect("response transforms");
+        assert_eq!(response.choices[0].message.content, Some(String::new()));
+    }
+}
+
+#[test]
+fn reports_the_total_tokens_converse_sent_rather_than_recomputing_them() {
+    // Python reads `usage["totalTokens"]` straight through here, where Anthropic
+    // has no such field and adds the two counts instead. The two agree while the
+    // gate declines every cache_control request, so this is what keeps them
+    // agreeing if that ever widens.
+    let response = transform_response(json!({
+        "output": {"message": {"content": [{"text": "x"}]}},
+        "stopReason": "end_turn",
+        "usage": {"inputTokens": 10, "outputTokens": 4, "cacheReadInputTokens": 7, "totalTokens": 14}
+    }))
+    .expect("response transforms");
+    assert_eq!(
+        response.usage.total_tokens, 14,
+        "provider total was recomputed"
+    );
+    assert_eq!(response.usage.prompt_tokens, 17);
+    assert_eq!(response.usage.completion_tokens, 4);
+}
+
+#[test]
+fn falls_back_to_the_computed_total_when_converse_omits_it() {
+    // Python raises a KeyError on a body with no `totalTokens`. Reporting a zero
+    // instead would be a worse divergence than the one above, so the computed
+    // total stands in.
+    let response = transform_response(json!({
+        "output": {"message": {"content": [{"text": "x"}]}},
+        "stopReason": "end_turn",
+        "usage": {"inputTokens": 10, "outputTokens": 4}
+    }))
+    .expect("response transforms");
+    assert_eq!(response.usage.total_tokens, 14);
+}
+
+#[test]
+fn declines_a_cache_control_message_so_widening_the_gate_is_a_red_test() {
+    // Converse only reports cache token counts when the request carries a
+    // cachePoint block, which is why the provider total and the computed one
+    // cannot disagree today. This is the tripwire: whoever widens the gate to
+    // admit prompt caching has to come back and re-check the usage mapping
+    // rather than discovering a silent number change in production.
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": [
+                {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}
+            ]}]),
+            json!({})
+        ),
+        Some(Unsupported("non-text message content"))
+    );
+}
+
+#[test]
+fn folds_converse_cache_tokens_into_prompt_tokens() {
+    let response = transform_response(json!({
+        "output": {"message": {"content": [{"text": "x"}]}},
+        "stopReason": "end_turn",
+        "usage": {
+            "inputTokens": 10,
+            "outputTokens": 2,
+            "cacheReadInputTokens": 5,
+            "cacheWriteInputTokens": 3
+        }
+    }))
+    .expect("response transforms");
+    assert_eq!(response.usage.prompt_tokens, 18);
+    assert_eq!(response.usage.prompt_tokens_details.cached_tokens, 5);
+    assert_eq!(
+        response.usage.prompt_tokens_details.cache_creation_tokens,
+        3
+    );
+    assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10);
+}
+
+#[test]
+fn declines_a_response_carrying_a_tool_use_block() {
+    let err = transform_response(json!({
+        "output": {"message": {"content": [
+            {"toolUse": {"toolUseId": "t1", "name": "f", "input": {}}}
+        ]}},
+        "stopReason": "tool_use",
+        "usage": {"inputTokens": 1, "outputTokens": 1}
+    }))
+    .expect_err("tool use block");
+    assert_eq!(
+        err,
+        CoreError::Unsupported("non-text response content block")
+    );
+}
+
+#[test]
+fn errors_on_a_response_missing_required_fields() {
+    assert_eq!(
+        transform_response(json!("nope")).expect_err("not an object"),
+        CoreError::InvalidResponse("converse response is not an object".to_string())
+    );
+    assert_eq!(
+        transform_response(json!({"usage": {}})).expect_err("no output"),
+        CoreError::MissingField("output.message.content")
+    );
+    assert_eq!(
+        transform_response(json!({"output": {"message": {"content": []}}})).expect_err("no usage"),
+        CoreError::MissingField("usage")
+    );
+}
+
+#[test]
+fn accepts_aws_call_configuration_without_serializing_it() {
+    let call_config = json!({
+        "maxTokens": 16,
+        "aws_access_key_id": "AKIA",
+        "aws_secret_access_key": "secret",
+        "aws_session_token": "token",
+        "aws_region_name": "us-east-1",
+        "aws_profile_name": "litellm-stage",
+        "aws_role_name": "role",
+        "aws_session_name": "session",
+        "aws_web_identity_token": "wit",
+        "aws_sts_endpoint": "https://sts.example",
+        "aws_external_id": "ext",
+        "aws_bedrock_runtime_endpoint": "https://vpce.internal"
+    });
+    assert_eq!(
+        reason(
+            json!([{"role": "user", "content": "hi"}]),
+            call_config.clone()
+        ),
+        None
+    );
+    let body = transform(json!([{"role": "user", "content": "hi"}]), call_config);
+    assert_eq!(
+        body,
+        json!({
+            "inferenceConfig": {"maxTokens": 16},
+            "messages": [{"role": "user", "content": [{"text": "hi"}]}]
+        }),
+        "aws call configuration must not reach the Converse body"
+    );
+}
+
+#[test]
+fn leaves_a_complete_converse_url_untouched() {
+    let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG;
+    let already_built =
+        "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-v2%3A0/converse";
+    assert_eq!(
+        config
+            .complete_url(
+                Some(already_built),
+                "anthropic.claude-v2",
+                &Map::new(),
+                &|_| None
+            )
+            .expect("url builds"),
+        already_built,
+        "a host that encoded the model id itself must not have it re-derived"
+    );
+}
+
+#[test]
+fn host_supplied_credentials_outrank_ambient_profile_and_role_state() {
+    use crate::providers::bedrock::aws_base::host_supplied_credentials;
+
+    let supplied = params(json!({
+        "aws_access_key_id": "AKIAHOST",
+        "aws_secret_access_key": "hostsecret",
+        "aws_session_token": "hosttoken"
+    }));
+    let credentials = host_supplied_credentials(&supplied).expect("host credentials");
+    assert_eq!(credentials.access_key_id(), "AKIAHOST");
+    assert_eq!(credentials.secret_access_key(), "hostsecret");
+    assert_eq!(credentials.session_token(), Some("hosttoken"));
+
+    // Without a full static pair there is nothing to honor, so the core falls
+    // back to deriving credentials itself.
+    assert!(host_supplied_credentials(&params(json!({"aws_access_key_id": "AKIA"}))).is_none());
+    assert!(
+        host_supplied_credentials(&params(
+            json!({"aws_access_key_id": "  ", "aws_secret_access_key": "s"})
+        ))
+        .is_none()
+    );
+    assert!(host_supplied_credentials(&Map::new()).is_none());
+}

litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs

@@ -0,0 +1,297 @@
+use serde_json::{Map, Value, json};
+
+use crate::chat_completions::conversation::{Conversation, TurnRole, build_conversation};
+use crate::chat_completions::response_utils::{finish_reason_for, unix_now, usage_from_parts};
+use crate::chat_completions::transformation::{
+    ChatCompletionsAuth, ChatCompletionsProviderConfig, Unsupported, unsupported_message,
+    unsupported_param,
+};
+use crate::chat_completions::types::{
+    ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse,
+    ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData,
+    ProviderChatResponseData,
+};
+use crate::error::{CoreError, CoreResult};
+
+use super::super::aws_base::{bedrock_model_id_and_region, resolve_bedrock_region};
+use super::super::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE};
+
+/// Converse parameter names, post `map_openai_params`, that the Rust path can
+/// place verbatim in `inferenceConfig`.
+///
+/// `topK` is deliberately absent: Python routes it to
+/// `additionalModelRequestFields` for Anthropic base models and to
+/// `inferenceConfig` otherwise, and that branch reads the model catalog the
+/// core cannot see.
+const SUPPORTED_PARAMS: &[&str] = &["maxTokens", "temperature", "topP", "stopSequences"];
+
+/// Params that belong in `inferenceConfig`, in the order Python's
+/// `AmazonConverseConfig` declares them, so bodies compare cleanly.
+const INFERENCE_CONFIG_PARAMS: &[&str] = SUPPORTED_PARAMS;
+
+const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = "aws_bedrock_runtime_endpoint";
+
+/// AWS call configuration a host passes down: consumed for signing and endpoint
+/// resolution, never serialized into the Converse body.
+const CONFIG_PARAMS: &[&str] = &[
+    "aws_access_key_id",
+    "aws_secret_access_key",
+    "aws_session_token",
+    "aws_region_name",
+    "aws_session_name",
+    "aws_profile_name",
+    "aws_role_name",
+    "aws_web_identity_token",
+    "aws_sts_endpoint",
+    "aws_external_id",
+    AWS_BEDROCK_RUNTIME_ENDPOINT,
+];
+
+const CONVERSE_PATH_SUFFIX: &str = "/converse";
+
+pub struct BedrockChatCompletionsConfig;
+
+pub const BEDROCK_CHAT_COMPLETIONS_CONFIG: BedrockChatCompletionsConfig =
+    BedrockChatCompletionsConfig;
+
+fn converse_body(conversation: &Conversation, params: &Map<String, Value>) -> Value {
+    let messages: Vec<Value> = conversation
+        .turns
+        .iter()
+        .map(|turn| {
+            json!({
+                "role": turn.role.as_str(),
+                "content": turn.texts.iter().map(|text| json!({"text": text})).collect::<Vec<_>>(),
+            })
+        })
+        .collect();
+
+    let inference_config = Map::from_iter(INFERENCE_CONFIG_PARAMS.iter().filter_map(|name| {
+        params
+            .get(*name)
+            .map(|value| ((*name).to_string(), value.clone()))
+    }));
+
+    let system: Vec<Value> = conversation
+        .system
+        .iter()
+        .map(|text| json!({"text": text}))
+        .collect();
+
+    Value::Object(Map::from_iter(
+        [
+            (
+                "inferenceConfig".to_string(),
+                Value::Object(inference_config),
+            ),
+            ("messages".to_string(), json!(messages)),
+        ]
+        .into_iter()
+        .chain((!system.is_empty()).then(|| ("system".to_string(), json!(system)))),
+    ))
+}
+
+fn has_blank_text(message: &ChatMessage) -> bool {
+    match &message.content {
+        None => false,
+        Some(ChatMessageContent::Text(text)) => text.trim().is_empty(),
+        Some(ChatMessageContent::Parts(parts)) => parts.iter().any(|part| {
+            part.get("text")
+                .and_then(Value::as_str)
+                .is_none_or(|text| text.trim().is_empty())
+        }),
+    }
+}
+
+impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig {
+    fn complete_url(
+        &self,
+        api_base: Option<&str>,
+        model: &str,
+        optional_params: &Map<String, Value>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<String> {
+        let (model_id, model_region) = bedrock_model_id_and_region(model);
+        let region = resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup);
+        let endpoint = optional_params
+            .get(AWS_BEDROCK_RUNTIME_ENDPOINT)
+            .and_then(Value::as_str)
+            .or(api_base)
+            .map(str::trim)
+            .filter(|value| !value.is_empty())
+            .map(str::to_string)
+            .unwrap_or_else(|| BEDROCK_RUNTIME_ENDPOINT_TEMPLATE.replace("{region}", &region));
+        let endpoint = endpoint.trim_end_matches('/');
+        // A host that already built the full Converse URL (LiteLLM's Python
+        // path encodes the model id itself) passes it through untouched, the
+        // way the Anthropic config leaves a complete `/v1/messages` URL alone.
+        if endpoint.ends_with(CONVERSE_PATH_SUFFIX) {
+            return Ok(endpoint.to_string());
+        }
+        Ok(format!("{endpoint}/model/{model_id}{CONVERSE_PATH_SUFFIX}"))
+    }
+
+    fn auth(
+        &self,
+        api_key: Option<&str>,
+        model: &str,
+        optional_params: &Map<String, Value>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> CoreResult<ChatCompletionsAuth> {
+        // Python reads `api_key` as the Bedrock bearer token and consults the
+        // env only when the caller passed none, so a caller-supplied empty key
+        // falls through to SigV4 without reaching for the environment. An
+        // all-whitespace token stays a bearer token here because Python sends
+        // it too: treating it as absent would sign as the host principal
+        // instead, which is the identity swap this branch exists to prevent.
+        let bearer = match api_key {
+            Some(key) => Some(key.to_string()),
+            None => env_lookup(AWS_BEARER_TOKEN_BEDROCK),
+        }
+        .filter(|token| !token.is_empty());
+        if let Some(token) = bearer {
+            return Ok(ChatCompletionsAuth::Bearer { token });
+        }
+        let (_, model_region) = bedrock_model_id_and_region(model);
+        Ok(ChatCompletionsAuth::AwsSigV4 {
+            region: resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup),
+        })
+    }
+
+    fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
+        &[("Content-Type", "application/json")]
+    }
+
+    fn supported_params(&self) -> &'static [&'static str] {
+        SUPPORTED_PARAMS
+    }
+
+    fn config_params(&self) -> &'static [&'static str] {
+        CONFIG_PARAMS
+    }
+
+    fn unsupported_reason(
+        &self,
+        messages: &[ChatMessage],
+        optional_params: &Map<String, Value>,
+    ) -> Option<Unsupported> {
+        unsupported_param(SUPPORTED_PARAMS, CONFIG_PARAMS, optional_params)
+            .or_else(|| messages.iter().find_map(unsupported_message))
+            // Python's Converse translation drops blank text blocks instead of
+            // substituting the placeholder the shared conversation builder
+            // applies, so decline blank text rather than diverge.
+            .or_else(|| {
+                messages
+                    .iter()
+                    .any(has_blank_text)
+                    .then_some(Unsupported("blank message text"))
+            })
+            // Converse has no assistant prefill: Python inserts a continue turn
+            // when a conversation opens or closes on an assistant message, and
+            // only under `litellm.modify_params`, which the core cannot see.
+            // Declining both ends also keeps the shared builder's final
+            // assistant right-strip (an Anthropic rule) unreachable here.
+            .or_else(|| {
+                let conversation = build_conversation(messages);
+                let ends_on_assistant = conversation
+                    .turns
+                    .last()
+                    .is_some_and(|turn| turn.role == TurnRole::Assistant);
+                (!conversation.opens_on_user_turn() || ends_on_assistant).then_some(Unsupported(
+                    "conversation does not run user turn to user turn",
+                ))
+            })
+    }
+
+    fn transform_request(
+        &self,
+        _model: &str,
+        messages: Vec<ChatMessage>,
+        optional_params: Map<String, Value>,
+    ) -> CoreResult<ProviderChatRequestData> {
+        Ok(ProviderChatRequestData {
+            body: converse_body(&build_conversation(&messages), &optional_params),
+        })
+    }
+
+    fn transform_response(
+        &self,
+        model: &str,
+        response: ProviderChatResponseData,
+    ) -> CoreResult<ChatCompletionsResponse> {
+        let body = response.body.as_object().ok_or_else(|| {
+            CoreError::InvalidResponse("converse response is not an object".into())
+        })?;
+
+        let content = body
+            .get("output")
+            .and_then(|output| output.get("message"))
+            .and_then(|message| message.get("content"))
+            .and_then(Value::as_array)
+            .ok_or(CoreError::MissingField("output.message.content"))?;
+        // The route declines tool requests, so anything other than a text block
+        // is something this path never asked for. Decline; the host falls back.
+        if content.iter().any(|block| {
+            block
+                .as_object()
+                .is_none_or(|block| block.len() != 1 || !block.contains_key("text"))
+        }) {
+            return Err(CoreError::Unsupported("non-text response content block"));
+        }
+        let text: String = content
+            .iter()
+            .filter_map(|block| block.get("text").and_then(Value::as_str))
+            .collect();
+
+        let usage = body
+            .get("usage")
+            .and_then(Value::as_object)
+            .ok_or(CoreError::MissingField("usage"))?;
+        let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0);
+        let computed = usage_from_parts(
+            field("inputTokens"),
+            field("outputTokens"),
+            field("cacheReadInputTokens"),
+            field("cacheWriteInputTokens"),
+        );
+        // Converse reports `totalTokens` and Python passes it straight through,
+        // where Anthropic has no such field and Python adds the two counts
+        // instead, so only this provider overrides the computed total. Python
+        // does a bare `usage["totalTokens"]` lookup, so a body without the key
+        // raises there rather than reporting a zero; fall back to the computed
+        // total, which is the closest thing to that without failing the call.
+        let usage = ChatCompletionsUsage {
+            total_tokens: usage
+                .get("totalTokens")
+                .and_then(Value::as_u64)
+                .unwrap_or(computed.total_tokens),
+            ..computed
+        };
+
+        Ok(ChatCompletionsResponse {
+            created: unix_now(),
+            // Converse echoes no model id, so Python reports the requested one.
+            model: model.to_string(),
+            choices: vec![ChatCompletionsChoice {
+                index: 0,
+                message: ChatCompletionsChoiceMessage {
+                    role: "assistant".to_string(),
+                    // Converse assigns the joined string unconditionally, so an
+                    // empty response is `""` here and not `None` as it is on
+                    // Anthropic. A caller calling `.strip()` on it would break
+                    // on this path alone.
+                    content: Some(text),
+                },
+                finish_reason: finish_reason_for(
+                    body.get("stopReason").and_then(Value::as_str).unwrap_or(""),
+                )
+                .to_string(),
+            }],
+            usage,
+        })
+    }
+}
+
+#[cfg(test)]
+#[path = "tests.rs"]
+mod tests;

litellm-rust/crates/core/src/providers/bedrock/constants.rs

@@ -11,6 +11,31 @@ pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN";
 pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
 pub const AWS_STS_ENDPOINT: &str = "AWS_STS_ENDPOINT";
 pub const AWS_EXTERNAL_ID: &str = "AWS_EXTERNAL_ID";
+pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK";
+
+/// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors
+/// Python's `_filter_headers_for_aws_signature` allowlist.
+pub const AWS_SIGNED_HEADER_NAMES: &[&str] = &[
+    "host",
+    "content-type",
+    "date",
+    "x-amz-date",
+    "x-amz-security-token",
+    "x-amz-content-sha256",
+    "x-amz-algorithm",
+    "x-amz-credential",
+    "x-amz-signedheaders",
+    "x-amz-signature",
+];
+/// Headers the signer emits itself. Mirrors Python's `SIGV4_COMPUTED_HEADERS`,
+/// which the reattach loop skips so a caller's copy cannot ride alongside the
+/// computed one.
+pub const SIGV4_COMPUTED_HEADER_NAMES: &[&str] = &[
+    "authorization",
+    "x-amz-date",
+    "x-amz-security-token",
+    "date",
+];
 pub const BEDROCK_SERVICE: &str = "bedrock";
 pub const DEFAULT_SESSION_NAME_PREFIX: &str = "litellm-session";
 pub const DEFAULT_BEDROCK_REGION: &str = "us-west-2";

litellm-rust/crates/core/src/providers/bedrock/mod.rs

@@ -5,4 +5,5 @@
 #[cfg(feature = "bedrock-auth")]
 pub mod audio_transcription;
 pub mod aws_base;
+pub mod chat_completions;
 mod constants;

litellm-rust/crates/python-bridge/src/lib.rs

@@ -6,6 +6,10 @@ use litellm_ai_gateway::io::audio_transcription::{
 };
 use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr};
 use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection;
+use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse};
+use litellm_core::chat_completions::{
+    chat_completions as run_chat_completions, chat_completions_decline_reason,
+};
 use litellm_core::error::CoreError;
 use litellm_core::messages::messages as run_messages;
 use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest};
@@ -16,6 +20,20 @@ use serde_json::{Map, Value};
 
 mod gil;
 
+pyo3::create_exception!(
+    _native,
+    RustBridgeDeclined,
+    pyo3::exceptions::PyException,
+    "The route declined before calling the provider, so the host may retry on its own path."
+);
+
+pyo3::create_exception!(
+    _native,
+    RustUpstreamError,
+    pyo3::exceptions::PyException,
+    "The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response."
+);
+
 type MarshaledOcrInputs = (
     Value,
     Option<Map<String, Value>>,
@@ -45,6 +63,15 @@ fn messages_response_to_py(
     json_to_py(py, value)
 }
 
+fn chat_completions_response_to_py(
+    py: Python<'_>,
+    response: ChatCompletionsResponse,
+) -> PyResult<Py<PyAny>> {
+    let value =
+        serde_json::to_value(response).map_err(|err| PyValueError::new_err(err.to_string()))?;
+    json_to_py(py, value)
+}
+
 fn core_error_to_pyerr(err: CoreError) -> PyErr {
     match err {
         CoreError::Auth(message) => PyValueError::new_err(message),
@@ -56,6 +83,33 @@ fn core_error_to_pyerr(err: CoreError) -> PyErr {
     }
 }
 
+/// Map a core error for a route whose host keeps a Python implementation.
+///
+/// The distinction the host needs is whether the provider was already called.
+/// Everything raised before the request goes out is safe for the host to retry
+/// on its own path; anything after it is not, because the provider has already
+/// done the work and billed for it.
+fn chat_completions_error_to_pyerr(err: CoreError) -> PyErr {
+    match err {
+        CoreError::Unsupported(_)
+        | CoreError::Auth(_)
+        | CoreError::InvalidProvider(_)
+        | CoreError::InvalidRequest(_)
+        | CoreError::InvalidType { .. }
+        | CoreError::MissingField(_)
+        | CoreError::Routing(_)
+        // Nothing reached the provider, so serving it on Python cannot double
+        // bill and is the only way the caller gets an answer at all.
+        | CoreError::Connect(_) => RustBridgeDeclined::new_err(err.to_string()),
+        CoreError::Http { status, body } => {
+            RustUpstreamError::new_err((status, format!("{status}: {body}")))
+        }
+        CoreError::Network(message) | CoreError::InvalidResponse(message) => {
+            RustUpstreamError::new_err((0u16, message))
+        }
+    }
+}
+
 fn optional_object_to_map(
     py: Python<'_>,
     name: &'static str,
@@ -430,6 +484,143 @@ fn amessages(
     })
 }
 
+type MarshaledChatCompletionsInputs = (
+    Value,
+    Map<String, Value>,
+    Option<Map<String, Value>>,
+    Option<Duration>,
+);
+
+fn marshal_chat_completions_inputs(
+    py: Python<'_>,
+    messages: Py<PyAny>,
+    optional_params: Option<Py<PyAny>>,
+    extra_headers: Option<Py<PyAny>>,
+    timeout_seconds: Option<f64>,
+) -> PyResult<MarshaledChatCompletionsInputs> {
+    let messages = py_to_json(py, messages.bind(py))?;
+    if !messages.is_array() {
+        return Err(PyValueError::new_err("messages must be a list"));
+    }
+    let optional_params = optional_object_to_map(py, "optional_params", optional_params)?;
+    let extra_headers = match extra_headers {
+        Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?),
+        None => None,
+    };
+    Ok((
+        messages,
+        optional_params,
+        extra_headers,
+        optional_timeout(timeout_seconds),
+    ))
+}
+
+/// The decline reason for this request, or `None` when the Rust path accepts
+/// it. Resolves no credentials and performs no I/O, so a host can ask before
+/// committing to either path.
+#[pyfunction]
+#[pyo3(signature = (model, messages, optional_params=None, custom_llm_provider=None))]
+fn chat_completions_decline(
+    py: Python<'_>,
+    model: String,
+    messages: Py<PyAny>,
+    optional_params: Option<Py<PyAny>>,
+    custom_llm_provider: Option<String>,
+) -> PyResult<Option<String>> {
+    let messages = py_to_json(py, messages.bind(py))?;
+    let optional_params = optional_object_to_map(py, "optional_params", optional_params)?;
+    Ok(chat_completions_decline_reason(
+        &model,
+        custom_llm_provider.as_deref(),
+        messages,
+        &optional_params,
+    )
+    .map(str::to_string))
+}
+
+#[pyfunction]
+#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))]
+#[allow(clippy::too_many_arguments)]
+fn chat_completions(
+    py: Python<'_>,
+    model: String,
+    messages: Py<PyAny>,
+    optional_params: Option<Py<PyAny>>,
+    api_key: Option<String>,
+    api_base: Option<String>,
+    custom_llm_provider: Option<String>,
+    extra_headers: Option<Py<PyAny>>,
+    timeout_seconds: Option<f64>,
+) -> PyResult<Py<PyAny>> {
+    let (messages, optional_params, extra_headers, timeout) = marshal_chat_completions_inputs(
+        py,
+        messages,
+        optional_params,
+        extra_headers,
+        timeout_seconds,
+    )?;
+
+    let result = gil::release_gil(py, || {
+        pyo3_async_runtimes::tokio::get_runtime().block_on(run_chat_completions(
+            ChatCompletionsRequest {
+                model: &model,
+                messages,
+                optional_params,
+                api_key: api_key.as_deref(),
+                api_base: api_base.as_deref(),
+                custom_llm_provider: custom_llm_provider.as_deref(),
+                extra_headers,
+                timeout,
+            },
+        ))
+    });
+
+    match result {
+        Ok(response) => chat_completions_response_to_py(py, response),
+        Err(err) => Err(chat_completions_error_to_pyerr(err)),
+    }
+}
+
+#[pyfunction]
+#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))]
+#[allow(clippy::too_many_arguments)]
+fn achat_completions(
+    py: Python<'_>,
+    model: String,
+    messages: Py<PyAny>,
+    optional_params: Option<Py<PyAny>>,
+    api_key: Option<String>,
+    api_base: Option<String>,
+    custom_llm_provider: Option<String>,
+    extra_headers: Option<Py<PyAny>>,
+    timeout_seconds: Option<f64>,
+) -> PyResult<Bound<'_, PyAny>> {
+    let (messages, optional_params, extra_headers, timeout) = marshal_chat_completions_inputs(
+        py,
+        messages,
+        optional_params,
+        extra_headers,
+        timeout_seconds,
+    )?;
+
+    pyo3_async_runtimes::tokio::future_into_py(py, async move {
+        let response = run_chat_completions(ChatCompletionsRequest {
+            model: &model,
+            messages,
+            optional_params,
+            api_key: api_key.as_deref(),
+            api_base: api_base.as_deref(),
+            custom_llm_provider: custom_llm_provider.as_deref(),
+            extra_headers,
+            timeout,
+        })
+        .await
+        .map_err(chat_completions_error_to_pyerr)?;
+
+        Python::attach(|py| chat_completions_response_to_py(py, response))
+    })
+}
+
 #[pyfunction]
 fn gil_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
     let stats = PyDict::new(py);
@@ -439,12 +630,18 @@ fn gil_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
 
 #[pymodule]
 fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    let py = module.py();
     module.add_function(wrap_pyfunction!(ocr, module)?)?;
     module.add_function(wrap_pyfunction!(aocr, module)?)?;
     module.add_function(wrap_pyfunction!(transcription, module)?)?;
     module.add_function(wrap_pyfunction!(atranscription, module)?)?;
     module.add_function(wrap_pyfunction!(messages, module)?)?;
     module.add_function(wrap_pyfunction!(amessages, module)?)?;
+    module.add("RustBridgeDeclined", py.get_type::<RustBridgeDeclined>())?;
+    module.add("RustUpstreamError", py.get_type::<RustUpstreamError>())?;
+    module.add_function(wrap_pyfunction!(chat_completions_decline, module)?)?;
+    module.add_function(wrap_pyfunction!(chat_completions, module)?)?;
+    module.add_function(wrap_pyfunction!(achat_completions, module)?)?;
     module.add_class::<ResponsesWebSocketConnection>()?;
     module.add_function(wrap_pyfunction!(gil_stats, module)?)?;
     Ok(())

litellm/__init__.py

@@ -453,6 +453,7 @@ max_end_user_budget_id: Optional[str] = None
 # backwards compatibility — arbitrary client-supplied identifiers still
 # pass through unchanged.
 validate_end_user_id_in_db: bool = False
+block_requests_for_models_without_pricing: bool = False
 disable_end_user_cost_tracking: Optional[bool] = None
 disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
 enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None

litellm/_logging.py

@@ -8,6 +8,12 @@ from logging import Formatter
 from typing import Any, Final
 
 import litellm
+from litellm.constants import (
+    LITELLM_TRUNCATED_PAYLOAD_FIELD,
+    LITELLM_TRUNCATION_STDOUT_SAFEGUARD_NOTE,
+    MAX_STRING_LENGTH_STDOUT_LOG,
+)
+from litellm.litellm_core_utils.env_utils import get_env_int
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.secret_redaction import redact_string, redact_structured_value
@@ -101,7 +107,7 @@ class SecretRedactionFilter(logging.Filter):
         # Redact exception tracebacks
         if record.exc_info and record.exc_info[1] is not None:
             try:
-                record.exc_text = _redact_string(self._formatter.formatException(record.exc_info))
+                record.exc_text = _redact_string(record.exc_text or self._formatter.formatException(record.exc_info))
             except Exception:
                 pass
 
@@ -116,6 +122,72 @@ class SecretRedactionFilter(logging.Filter):
 _secret_filter: Final = SecretRedactionFilter()
 
 
+def _get_max_string_length_stdout_log() -> int:
+    """Read the limit per record so a value loaded later via proxy config
+    environment_variables is honored."""
+    return get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", MAX_STRING_LENGTH_STDOUT_LOG)
+
+
+def _stdout_truncation_marker(skipped_chars: int) -> str:
+    return (
+        f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. "
+        f"{LITELLM_TRUNCATION_STDOUT_SAFEGUARD_NOTE}) ..."
+    )
+
+
+def _truncate_for_stdout_log(text: str, limit: int) -> str:
+    kept_chars: Final = limit - len(_stdout_truncation_marker(len(text)))
+    if kept_chars <= 0:
+        return text[:limit]
+    head_chars: Final = kept_chars // 2
+    tail_chars: Final = kept_chars - head_chars
+    return f"{text[:head_chars]}{_stdout_truncation_marker(len(text) - kept_chars)}{text[-tail_chars:]}"
+
+
+class StdoutLogTruncationFilter(logging.Filter):
+    """Bounds how much of an oversized log line reaches stdout.
+
+    A provider error string can echo the whole request payload, so one failed agentic
+    request writes hundreds of KB to stdout, repeatedly as the exception propagates from
+    the router to the proxy handler and into its traceback, all inline on the event loop.
+
+    DEBUG records pass through untouched, since dumping full payloads is the point of
+    `--detailed_debug`, and logging callbacks (OTEL, Datadog, etc.) don't run through
+    logging filters at all, so they still get the untruncated error.
+    """
+
+    _formatter = logging.Formatter()
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if record.levelno < logging.INFO:
+            return True
+
+        limit: Final = _get_max_string_length_stdout_log()
+        if limit <= 0:
+            return True
+
+        try:
+            message: Final = record.getMessage()
+        except (TypeError, ValueError):
+            return True
+
+        if len(message) > limit:
+            record.msg = _truncate_for_stdout_log(message, limit)  # rebind-ok: the Filter interface mutates the record
+            record.args = None  # rebind-ok: args are consumed by the truncated message above
+
+        if isinstance(record.exc_info, tuple):
+            exc_text: Final = record.exc_text or self._formatter.formatException(record.exc_info)
+            if len(exc_text) > limit:
+                record.exc_text = _truncate_for_stdout_log(  # rebind-ok: the Filter interface mutates the record
+                    exc_text, limit
+                )
+
+        return True
+
+
+_stdout_truncation_filter: Final = StdoutLogTruncationFilter()
+
+
 class CorrelationContextFilter(logging.Filter):
     """Stamps each log record with the current request's trace_id and session_id from contextvars.
 
@@ -301,6 +373,7 @@ def _setup_json_exception_handlers(formatter):
     error_handler: Final = logging.StreamHandler()
     error_handler.setFormatter(formatter)
     error_handler.addFilter(_secret_filter)
+    error_handler.addFilter(_stdout_truncation_filter)
     error_handler.addFilter(_correlation_filter)
 
     # Setup excepthook for uncaught exceptions
@@ -365,6 +438,12 @@ verbose_router_logger.addHandler(handler)
 verbose_proxy_logger.addHandler(handler)
 verbose_logger.addHandler(handler)
 
+# Filters attached to the logger, not the handler, survive callers swapping in their own
+# handlers (JSON mode, uvicorn log config, a host app's root handler).
+verbose_router_logger.addFilter(_stdout_truncation_filter)
+verbose_proxy_logger.addFilter(_stdout_truncation_filter)
+verbose_logger.addFilter(_stdout_truncation_filter)
+
 
 def _suppress_loggers():
     """Suppress noisy loggers at INFO level"""

litellm/_redis.py

@@ -17,6 +17,7 @@ from typing import Final
 
 import redis
 import redis.asyncio as async_redis
+from redis.credentials import CredentialProvider
 
 from litellm import get_secret, get_secret_str
 from litellm._redis_credential_provider import (
@@ -134,6 +135,7 @@ def _get_redis_cluster_kwargs(client=None):
         "ssl_check_hostname",
         "ssl_ca_certs",
         "redis_connect_func",  # Needed for sync clusters and IAM detection
+        "credential_provider",
         "gcp_service_account",
         "gcp_ssl_ca_certs",
         "azure_redis_ad_token",
@@ -549,14 +551,22 @@ def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
     return sentinel.master_for(service_name, **connection_kwargs)
 
 
+def _sentinel_auth_kwargs(connection_kwargs: dict, sentinel_password: str | None) -> dict:
+    """The Sentinel monitors are separate servers that authenticate with their own password, so the
+    data node's credential provider never belongs on them: leaving it there makes redis-py send the
+    data node's token to a monitor, which fails whether the monitor is unauthenticated or has its
+    own password."""
+    kept: Final = ((k, v) for k, v in connection_kwargs.items() if k != "credential_provider")
+    return dict(kept, password=sentinel_password)
+
+
 def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
     sentinel_nodes: Final = redis_kwargs.get("sentinel_nodes")
     sentinel_password: Final = redis_kwargs.get("sentinel_password")
     service_name: Final = redis_kwargs.get("service_name")
     connection_kwargs: Final = _get_redis_sentinel_connection_kwargs(redis_kwargs)
     connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
-    sentinel_kwargs: Final = dict(connection_kwargs)
-    sentinel_kwargs["password"] = sentinel_password
+    sentinel_kwargs: Final = _sentinel_auth_kwargs(connection_kwargs, sentinel_password)
 
     if not sentinel_nodes or not service_name:
         raise ValueError("Both 'sentinel_nodes' and 'service_name' are required for Redis Sentinel.")
@@ -574,6 +584,36 @@ def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
     return sentinel.master_for(service_name, **connection_kwargs)
 
 
+def _async_credential_provider(redis_connect_func: object | None) -> CredentialProvider | None:
+    """The Azure AD and GCP IAM connect funcs run their AUTH exchange with the blocking client
+    API, so on an async connection their ``send_command``/``read_response`` calls return
+    coroutines nobody awaits and every connect fails. Async paths authenticate through a
+    ``CredentialProvider`` instead, which redis-py consults per connection so the token stays
+    fresh. Any other ``redis_connect_func`` is left where it is, since redis-py awaits it
+    itself when it is a coroutine function."""
+    gcp_service_account: Final = getattr(redis_connect_func, "_gcp_service_account", None)
+    if gcp_service_account is not None:
+        return GCPIAMCredentialProvider(gcp_service_account)
+
+    azure_credential: Final = getattr(redis_connect_func, "_azure_credential", None)
+    if azure_credential is not None:
+        return AzureADCredentialProvider(azure_credential, username=os.environ.get("REDIS_USERNAME") or None)
+
+    return None
+
+
+def _async_auth_kwargs(redis_kwargs: dict) -> dict:
+    """Swaps a connect func an async path cannot run for the equivalent credential provider,
+    which supersedes any static username or password redis-py would otherwise reject it with."""
+    credential_provider: Final = _async_credential_provider(redis_kwargs.get("redis_connect_func"))
+    if credential_provider is None:
+        return redis_kwargs
+
+    superseded: Final = frozenset({"redis_connect_func", "username", "password"})
+    kept: Final = ((k, v) for k, v in redis_kwargs.items() if k not in superseded)
+    return dict(kept, credential_provider=credential_provider)  # mutable-ok: the branches below mutate these kwargs
+
+
 def get_redis_client(**env_overrides):
     redis_kwargs: Final = _get_redis_client_logic(**env_overrides)
 
@@ -600,7 +640,7 @@ def get_redis_async_client(
     connection_pool: async_redis.BlockingConnectionPool | None = None,
     **env_overrides,
 ) -> async_redis.Redis | async_redis.RedisCluster:
-    redis_kwargs: Final = _get_redis_client_logic(**env_overrides)
+    redis_kwargs: Final = _async_auth_kwargs(_get_redis_client_logic(**env_overrides))
 
     if "startup_nodes" in redis_kwargs:
         from redis.cluster import ClusterNode
@@ -611,28 +651,12 @@ def get_redis_async_client(
             if arg in args:
                 cluster_kwargs[arg] = redis_kwargs[arg]
 
-        # Handle GCP IAM authentication for async clusters
-        redis_connect_func = cluster_kwargs.pop("redis_connect_func", None)
-
-        # Use a CredentialProvider so the IAM token is regenerated on every new
-        # connection — mirrors the sync path where redis_connect_func is invoked
-        # per connection.  Without this, the token would expire after ~1 hour.
-        if redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
-            cluster_kwargs["credential_provider"] = GCPIAMCredentialProvider(redis_connect_func._gcp_service_account)
-        # Handle Azure AD authentication for async clusters via CredentialProvider
-        # so the credential's internal cache + silent refresh runs per connection
-        # (mirrors GCP IAM above; avoids static-token-baked-in-pool expiry).
-        elif redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
-            cluster_kwargs["credential_provider"] = AzureADCredentialProvider(
-                redis_connect_func._azure_credential,
-                username=os.environ.get("REDIS_USERNAME") or None,
-            )
-
         new_startup_nodes: Final[list[ClusterNode]] = []
 
         for item in redis_kwargs["startup_nodes"]:
             new_startup_nodes.append(ClusterNode(**item))
         cluster_kwargs.pop("startup_nodes", None)
+        cluster_kwargs.pop("redis_connect_func", None)
 
         # Default to a periodic health check + TCP keepalive so a connection silently dropped
         # by a cluster restart (e.g. ElastiCache Serverless maintenance) is revalidated and
@@ -667,19 +691,6 @@ def get_redis_async_client(
     if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs:
         return _init_async_redis_sentinel(redis_kwargs)
 
-    # Wrap GCP / Azure AD auth in a CredentialProvider for the standard async
-    # Redis client. The async client doesn't support redis_connect_func, but it
-    # does honour credential_provider — which is called per connection, so the
-    # underlying SDK can refresh tokens silently before they expire.
-    redis_connect_func = redis_kwargs.pop("redis_connect_func", None)
-    if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
-        redis_kwargs["credential_provider"] = AzureADCredentialProvider(
-            redis_connect_func._azure_credential,
-            username=os.environ.get("REDIS_USERNAME") or None,
-        )
-    elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
-        redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(redis_connect_func._gcp_service_account)
-
     _pretty_print_redis_config(redis_kwargs=redis_kwargs)
 
     if connection_pool is not None:
@@ -693,7 +704,7 @@ def get_redis_async_client(
 def get_redis_connection_pool(
     **env_overrides,
 ) -> async_redis.BlockingConnectionPool | None:
-    redis_kwargs: Final = _get_redis_client_logic(**env_overrides)
+    redis_kwargs: Final = _async_auth_kwargs(_get_redis_client_logic(**env_overrides))
     verbose_logger.debug("get_redis_connection_pool: redis_kwargs", redis_kwargs)
 
     if "startup_nodes" in redis_kwargs:
@@ -714,18 +725,6 @@ def get_redis_connection_pool(
                 )
         return async_redis.BlockingConnectionPool.from_url(**pool_kwargs)
 
-    # Wrap GCP / Azure AD auth in a CredentialProvider so pool-managed
-    # connections re-fetch tokens via the SDK's internal cache + silent refresh
-    # rather than reusing a single token captured at pool creation.
-    redis_connect_func: Final = redis_kwargs.pop("redis_connect_func", None)
-    if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
-        redis_kwargs["credential_provider"] = AzureADCredentialProvider(
-            redis_connect_func._azure_credential,
-            username=os.environ.get("REDIS_USERNAME") or None,
-        )
-    elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
-        redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(redis_connect_func._gcp_service_account)
-
     if redis_kwargs.pop("ssl", None):
         redis_kwargs["connection_class"] = async_redis.SSLConnection
     return async_redis.BlockingConnectionPool(timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs)

litellm/caching/_embedding_router.py

@@ -16,6 +16,7 @@ from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any, Final
 
 import litellm
+from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
 
 if TYPE_CHECKING:
     from litellm.router import Router
@@ -60,6 +61,13 @@ def resolve_embedding_max_input_tokens(
     return deployment_max_input_tokens
 
 
+def resolve_embedding_timeout(configured_timeout: float | None) -> float:
+    """Explicit cache setting first, else the short semantic-cache default."""
+    if configured_timeout is not None:
+        return configured_timeout
+    return SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
+
+
 def truncate_embedding_input(prompt: str, embedding_model: str, max_input_tokens: int | None) -> str:
     """Keep only the first ``max_input_tokens`` tokens of ``prompt`` for the embedding call."""
     if max_input_tokens is None:

litellm/caching/caching.py

@@ -98,6 +98,7 @@ class Cache:
         qdrant_semantic_cache_embedding_model: str = "text-embedding-ada-002",
         qdrant_semantic_cache_vector_size: int | None = None,
         semantic_cache_embedding_max_input_tokens: int | None = None,
+        semantic_cache_embedding_timeout: float | None = None,
         # GCP IAM authentication parameters
         gcp_service_account: str | None = None,
         gcp_ssl_ca_certs: str | None = None,
@@ -124,6 +125,7 @@ class Cache:
             qdrant_collection_name (str, optional): The name for your qdrant collection. Required if type is "qdrant-semantic".
             similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
             semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
+            semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
 
             # Disk Cache Args
             disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@@ -195,6 +197,7 @@ class Cache:
                 embedding_model=redis_semantic_cache_embedding_model,
                 index_name=redis_semantic_cache_index_name,
                 embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
+                embedding_timeout=semantic_cache_embedding_timeout,
                 **kwargs,
             )
         elif type == LiteLLMCacheType.VALKEY_SEMANTIC:
@@ -211,6 +214,7 @@ class Cache:
                 index_name=valkey_semantic_cache_index_name,
                 startup_nodes=redis_startup_nodes,
                 embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
+                embedding_timeout=semantic_cache_embedding_timeout,
                 **kwargs,
             )
         elif type == LiteLLMCacheType.QDRANT_SEMANTIC:
@@ -223,6 +227,7 @@ class Cache:
                 embedding_model=qdrant_semantic_cache_embedding_model,
                 vector_size=qdrant_semantic_cache_vector_size,
                 embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
+                embedding_timeout=semantic_cache_embedding_timeout,
             )
         elif type == LiteLLMCacheType.LOCAL:
             self.cache = InMemoryCache()

litellm/caching/qdrant_semantic_cache.py

@@ -16,7 +16,11 @@ from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import print_verbose
-from litellm.constants import QDRANT_SCALAR_QUANTILE, QDRANT_VECTOR_SIZE
+from litellm.constants import (
+    QDRANT_SCALAR_QUANTILE,
+    QDRANT_VECTOR_SIZE,
+    SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS,
+)
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_str_from_messages,
 )
@@ -26,6 +30,7 @@ from ._embedding_router import (
     build_router_embedding_metadata,
     resolve_embedding_max_input_tokens,
     resolve_embedding_router,
+    resolve_embedding_timeout,
     truncate_embedding_input,
 )
 from .base_cache import BaseCache
@@ -37,6 +42,7 @@ if TYPE_CHECKING:
 class QdrantSemanticCache(BaseCache):
     CACHE_KEY_FIELD_NAME = "litellm_cache_key"
     embedding_max_input_tokens: int | None = None
+    embedding_timeout: float = SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
 
     def __init__(
         self,
@@ -49,6 +55,7 @@ class QdrantSemanticCache(BaseCache):
         host_type=None,
         vector_size=None,
         embedding_max_input_tokens: int | None = None,
+        embedding_timeout: float | None = None,
     ):
         from litellm.llms.custom_httpx.http_handler import (
             _get_httpx_client,
@@ -68,6 +75,7 @@ class QdrantSemanticCache(BaseCache):
         self.similarity_threshold = similarity_threshold
         self.embedding_model = embedding_model
         self.embedding_max_input_tokens = embedding_max_input_tokens
+        self.embedding_timeout = resolve_embedding_timeout(embedding_timeout)
         self.vector_size = vector_size if vector_size is not None else QDRANT_VECTOR_SIZE
         headers = {}
 
@@ -222,11 +230,15 @@ class QdrantSemanticCache(BaseCache):
                 input=embedding_input,
                 cache={"no-store": True, "no-cache": True},
                 metadata=build_router_embedding_metadata(metadata),
+                timeout=self.embedding_timeout,
+                num_retries=0,
             )
         return litellm.embedding(
             model=self.embedding_model,
             input=embedding_input,
             cache={"no-store": True, "no-cache": True},
+            timeout=self.embedding_timeout,
+            num_retries=0,
         )
 
     async def _get_async_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> EmbeddingResponse:
@@ -238,19 +250,25 @@ class QdrantSemanticCache(BaseCache):
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
         embedding_input: Final = self._embedding_input(prompt, router)
-        if router is not None:
-            return await router.aembedding(
+        embedding_call: Final = (
+            router.aembedding(
                 model=self.embedding_model,
                 input=embedding_input,
                 cache={"no-store": True, "no-cache": True},
                 metadata=build_router_embedding_metadata(metadata),
+                timeout=self.embedding_timeout,
+                num_retries=0,
+            )
+            if router is not None
+            else litellm.aembedding(
+                model=self.embedding_model,
+                input=embedding_input,
+                cache={"no-store": True, "no-cache": True},
+                timeout=self.embedding_timeout,
+                num_retries=0,
             )
-
-        return await litellm.aembedding(
-            model=self.embedding_model,
-            input=embedding_input,
-            cache={"no-store": True, "no-cache": True},
         )
+        return await asyncio.wait_for(embedding_call, self.embedding_timeout)
 
     def set_cache(self, key, value, **kwargs):
         print_verbose(f"qdrant semantic-cache set_cache, kwargs: {kwargs}")

litellm/caching/redis_semantic_cache.py

@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
+from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_str_from_messages,
 )
@@ -27,6 +28,7 @@ from ._embedding_router import (
     build_router_embedding_metadata,
     resolve_embedding_max_input_tokens,
     resolve_embedding_router,
+    resolve_embedding_timeout,
     truncate_embedding_input,
 )
 from .base_cache import BaseCache
@@ -47,6 +49,7 @@ class RedisSemanticCache(BaseCache):
     DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index"
     CACHE_KEY_FIELD_NAME: str = "litellm_cache_key"
     embedding_max_input_tokens: int | None = None
+    embedding_timeout: float = SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
 
     def __init__(
         self,
@@ -58,6 +61,7 @@ class RedisSemanticCache(BaseCache):
         embedding_model: str = "text-embedding-ada-002",
         index_name: str | None = None,
         embedding_max_input_tokens: int | None = None,
+        embedding_timeout: float | None = None,
         **kwargs: object,
     ):
         """
@@ -74,6 +78,8 @@ class RedisSemanticCache(BaseCache):
             index_name: Name for the Redis index
             embedding_max_input_tokens: Truncate prompts to this many tokens before
                 embedding; defaults to the Router deployment's configured max_input_tokens
+            embedding_timeout: Seconds a cache lookup may spend embedding the prompt before it
+                gives up and lets the request continue to the LLM
             ttl: Default time-to-live for cache entries in seconds
             **kwargs: Additional arguments passed to the Redis client
 
@@ -99,6 +105,7 @@ class RedisSemanticCache(BaseCache):
         self.distance_threshold = 1 - similarity_threshold
         self.embedding_model = embedding_model
         self.embedding_max_input_tokens = embedding_max_input_tokens
+        self.embedding_timeout = resolve_embedding_timeout(embedding_timeout)
 
         # Set up Redis connection
         if redis_url is None:
@@ -349,6 +356,8 @@ class RedisSemanticCache(BaseCache):
                     input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
                     metadata=build_router_embedding_metadata(metadata),
+                    timeout=self.embedding_timeout,
+                    num_retries=0,
                 ),
             )
         else:
@@ -358,6 +367,8 @@ class RedisSemanticCache(BaseCache):
                     model=self.embedding_model,
                     input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
+                    timeout=self.embedding_timeout,
+                    num_retries=0,
                 ),
             )
         return embedding_response["data"][0]["embedding"]
@@ -512,20 +523,26 @@ class RedisSemanticCache(BaseCache):
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
         embedding_input: Final = self._embedding_input(prompt, router)
+        embedding_call: Final = (
+            router.aembedding(
+                model=self.embedding_model,
+                input=embedding_input,
+                cache={"no-store": True, "no-cache": True},
+                metadata=build_router_embedding_metadata(metadata),
+                timeout=self.embedding_timeout,
+                num_retries=0,
+            )
+            if router is not None
+            else litellm.aembedding(
+                model=self.embedding_model,
+                input=embedding_input,
+                cache={"no-store": True, "no-cache": True},
+                timeout=self.embedding_timeout,
+                num_retries=0,
+            )
+        )
         try:
-            if router is not None:
-                embedding_response = await router.aembedding(
-                    model=self.embedding_model,
-                    input=embedding_input,
-                    cache={"no-store": True, "no-cache": True},
-                    metadata=build_router_embedding_metadata(metadata),
-                )
-            else:
-                embedding_response = await litellm.aembedding(
-                    model=self.embedding_model,
-                    input=embedding_input,
-                    cache={"no-store": True, "no-cache": True},
-                )
+            embedding_response: Final = await asyncio.wait_for(embedding_call, self.embedding_timeout)
             return embedding_response["data"][0]["embedding"]
         except Exception as e:
             print_verbose(f"Error generating async embedding: {e}")

litellm/caching/valkey_semantic_cache.py

@@ -30,6 +30,7 @@ from litellm._logging import print_verbose
 from litellm._uuid import uuid
 from litellm.llms.valkey.common_utils import build_valkey_url, pack_vector
 
+from ._embedding_router import resolve_embedding_timeout
 from .redis_semantic_cache import RedisSemanticCache
 
 
@@ -62,6 +63,7 @@ class ValkeySemanticCache(RedisSemanticCache):
         sync_client: Redis | None = None,
         async_client: AsyncRedis | None = None,
         embedding_max_input_tokens: int | None = None,
+        embedding_timeout: float | None = None,
         **kwargs: Any,
     ):
         if similarity_threshold is None:
@@ -80,6 +82,7 @@ class ValkeySemanticCache(RedisSemanticCache):
         self.similarity_threshold = similarity_threshold
         self.embedding_model = embedding_model
         self.embedding_max_input_tokens = embedding_max_input_tokens
+        self.embedding_timeout = resolve_embedding_timeout(embedding_timeout)
         self.index_name = index_name or self.DEFAULT_VALKEY_INDEX_NAME
         self.key_prefix = f"{self.index_name}:"
         self._index_dim: int | None = None

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -25,6 +25,11 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
     encoding: object
 
 
+def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
+    """`responses()` runs `get_llm_provider()` itself, so hand back the prefixed model `completion()` started from."""
+    return f"{custom_llm_provider}/{model}"
+
+
 class ResponsesToCompletionBridgeHandler:
     def __init__(self):
         from .transformation import LiteLLMResponsesTransformationHandler
@@ -184,14 +189,11 @@ class ResponsesToCompletionBridgeHandler:
             client=kwargs.get("client"),
         )
 
-        # Pin the resolved provider so `responses()` doesn't re-run
-        # `get_llm_provider()` on the model string and strip a second
-        # provider prefix (see GitHub issue #28505).  request_data already
-        # carries `custom_llm_provider` via the spread of
-        # `sanitized_litellm_params`; overwriting it on the dict (rather
-        # than adding an explicit kwarg) avoids the duplicate-keyword
-        # TypeError that would otherwise fire on the real bridge path.
+        # Set on request_data rather than passed as explicit kwargs: the spread of
+        # `sanitized_litellm_params` already carries both, so passing them again
+        # would raise a duplicate-keyword TypeError.
         request_data["custom_llm_provider"] = custom_llm_provider
+        request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
         result: Final = responses(
             **request_data,
         )
@@ -282,13 +284,11 @@ class ResponsesToCompletionBridgeHandler:
         except Exception as e:
             raise e
 
-        # Pin the resolved provider so `aresponses()` doesn't re-run
-        # `get_llm_provider()` on the model string and strip a second
-        # provider prefix (see GitHub issue #28505).  Set on request_data
-        # rather than passed as a separate kwarg to avoid the duplicate-
-        # keyword TypeError when `sanitized_litellm_params` already
-        # carries `custom_llm_provider`.
+        # Set on request_data rather than passed as explicit kwargs: the spread of
+        # `sanitized_litellm_params` already carries both, so passing them again
+        # would raise a duplicate-keyword TypeError.
         request_data["custom_llm_provider"] = custom_llm_provider
+        request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
         result: Final = await aresponses(
             **request_data,
             aresponses=True,

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -113,6 +113,58 @@ def _build_reasoning_item(
     }
 
 
+def _reasoning_item_from_output_item(item: object) -> _BuiltReasoningItem | None:
+    from openai.types.responses import ResponseReasoningItem
+
+    if isinstance(item, ResponseReasoningItem):
+        return _build_reasoning_item(
+            item_id=item.id,
+            encrypted_content=getattr(item, "encrypted_content", None),
+            summary_raw=item.summary,
+        )
+    if isinstance(item, dict) and item.get("type") == "reasoning":
+        return _build_reasoning_item(
+            item_id=item.get("id", ""),
+            encrypted_content=item.get("encrypted_content"),
+            summary_raw=item.get("summary"),
+        )
+    return None
+
+
+def _reasoning_items_from_output_items(output_items: Sequence[object]) -> tuple[_BuiltReasoningItem, ...]:
+    return tuple(
+        reasoning_item
+        for reasoning_item in (_reasoning_item_from_output_item(item) for item in output_items)
+        if reasoning_item is not None
+    )
+
+
+def _as_chat_reasoning_items(
+    reasoning_items: Sequence[_BuiltReasoningItem],
+) -> list[ChatCompletionReasoningItem] | None:
+    if not reasoning_items:
+        return None
+    # cast-ok: _BuiltReasoningItem is the structural shape ChatCompletionReasoningItem
+    # describes, and TypedDict invariance is what stops the two from unifying here.
+    return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
+
+
+def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Literal["length", "content_filter"]:
+    if incomplete_reason == "content_filter":
+        return "content_filter"
+    return "length"
+
+
+def _incomplete_reason_from_response_payload(response_payload: object) -> str | None:
+    if not isinstance(response_payload, Mapping):
+        return None
+    incomplete_details: Final = response_payload.get("incomplete_details")
+    if not isinstance(incomplete_details, Mapping):
+        return None
+    reason: Final = incomplete_details.get("reason")
+    return reason if isinstance(reason, str) else None
+
+
 class _ChatToolCallDict(ChatCompletionToolCallChunk, total=False):
     provider_specific_fields: Mapping[str, object]
 
@@ -657,6 +709,27 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         return choices
 
+    @staticmethod
+    def _build_empty_incomplete_choice(
+        output_items: Sequence[object],
+        finish_reason: Literal["length", "content_filter"],
+    ) -> "Choices":
+        from litellm.types.utils import Choices, Message
+
+        reasoning_items: Final = _reasoning_items_from_output_items(output_items)
+        reasoning_content: Final = " ".join(
+            summary_block["text"]
+            for reasoning_item in reasoning_items
+            for summary_block in reasoning_item["summary"]
+            if summary_block.get("text")
+        )
+        message: Final = Message(
+            content="",
+            reasoning_content=reasoning_content if reasoning_content else None,
+            reasoning_items=_as_chat_reasoning_items(reasoning_items),
+        )
+        return Choices(message=message, finish_reason=finish_reason, index=0)
+
     @classmethod
     def _extract_output_from_completed_event(cls, parsed_chunk: Mapping[str, object]) -> list[dict[str, object]] | None:
         response_payload: Final = parsed_chunk.get("response")
@@ -763,11 +836,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             handle_raw_dict_callback=self._handle_raw_dict_response_item,
         )
 
-        if len(choices) == 0:
-            if raw_response.incomplete_details is not None and raw_response.incomplete_details.reason is not None:
-                raise ValueError(f"{model} unable to complete request: {raw_response.incomplete_details.reason}")
+        response_is_incomplete: Final = raw_response.status == "incomplete" or (
+            raw_response.incomplete_details is not None and raw_response.incomplete_details.reason is not None
+        )
+
+        if len(choices) == 0 and not response_is_incomplete:
+            raise ValueError(f"Unknown items in responses API response: {output_items}")
+
+        if response_is_incomplete:
+            incomplete_finish_reason: Final = _map_incomplete_reason_to_finish_reason(
+                raw_response.incomplete_details.reason if raw_response.incomplete_details is not None else None
+            )
+            if len(choices) == 0:
+                choices.append(self._build_empty_incomplete_choice(output_items, incomplete_finish_reason))
             else:
-                raise ValueError(f"Unknown items in responses API response: {output_items}")
+                for choice in choices:
+                    choice.finish_reason = incomplete_finish_reason
 
         setattr(model_response, "choices", choices)
 
@@ -1392,12 +1476,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                         )
                     ]
                 )
-        elif event_type == "response.completed":
-            # Response is fully complete - now we can signal is_finished=True
-            # This ensures we don't prematurely end the stream before tool_calls arrive
-
-            # Check if response contains function_call items in output
-            # to determine correct finish_reason
+        elif event_type in ("response.completed", "response.incomplete"):
             response_data: Final = parsed_chunk.get("response", {})
             output_items: Final = response_data.get("output", []) if response_data else []
 
@@ -1407,25 +1486,14 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                 if isinstance(item, dict)
             )
 
-            finish_reason: Final = "tool_calls" if has_function_calls else "stop"
+            finish_reason: Final = (
+                _map_incomplete_reason_to_finish_reason(_incomplete_reason_from_response_payload(response_data))
+                if event_type == "response.incomplete"
+                else ("tool_calls" if has_function_calls else "stop")
+            )
 
-            # Extract reasoning items with encrypted_content for round-tripping
-            completed_reasoning_items: list[_BuiltReasoningItem] | None = None
-            for item in output_items:
-                if not isinstance(item, dict) or item.get("type") != "reasoning":
-                    continue
-                if completed_reasoning_items is None:
-                    completed_reasoning_items = []
-                completed_reasoning_items.append(
-                    _build_reasoning_item(
-                        item_id=item.get("id", ""),
-                        encrypted_content=item.get("encrypted_content"),
-                        summary_raw=item.get("summary"),
-                    )
-                )
-            completed_reasoning_items_typed: Final = cast(
-                list[ChatCompletionReasoningItem] | None,
-                completed_reasoning_items,
+            terminal_reasoning_items_typed: Final = _as_chat_reasoning_items(
+                _reasoning_items_from_output_items(output_items)
             )
 
             usage = None
@@ -1439,7 +1507,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                         index=0,
                         delta=Delta(
                             content="",
-                            reasoning_items=completed_reasoning_items_typed,
+                            reasoning_items=terminal_reasoning_items_typed,
                         ),
                         finish_reason=finish_reason,
                     )

litellm/constants.py

@@ -3,7 +3,7 @@ import sys
 from types import MappingProxyType
 from typing import Final, Literal
 
-from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_or_none
+from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_in_range, get_env_int_or_none
 
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
@@ -49,6 +49,8 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
 # Set to 0 to disable truncation.
 MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
 
+MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)
+
 # When true, adds detailed per-phase timing breakdown headers to responses.
 # Headers: x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms
 LITELLM_DETAILED_TIMING: Final = os.getenv("LITELLM_DETAILED_TIMING", "false").lower() == "true"
@@ -323,6 +325,17 @@ DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_MOCK_RE
 DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", 20))
 MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES: Final = int(os.getenv("MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES", 768))
 MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES: Final = int(os.getenv("MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES", 2000))
+# tiktoken's BPE merge loop is quadratic in the length of a single regex piece, so a long run of one
+# repeated character (dot leaders, whitespace, zero-padded base64) can take minutes on a multi-MB payload.
+# Encoding in chunks makes the cost linear, at a drift of at most ~1 token per chunk boundary. The upper
+# bound keeps a misconfigured chunk size from restoring the quadratic cost this exists to remove.
+TIKTOKEN_ENCODE_MAX_CHUNK_SIZE_CHARS: Final = 4096
+TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS: Final = get_env_int_in_range(
+    "TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS",
+    default=1024,
+    minimum=1,
+    maximum=TIKTOKEN_ENCODE_MAX_CHUNK_SIZE_CHARS,
+)
 MAX_TILE_WIDTH: Final = int(os.getenv("MAX_TILE_WIDTH", 512))
 MAX_TILE_HEIGHT: Final = int(os.getenv("MAX_TILE_HEIGHT", 512))
 OPENAI_FILE_SEARCH_COST_PER_1K_CALLS: Final = float(os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000))
@@ -423,6 +436,9 @@ DEFAULT_REQUEST_TIMEOUT_SECONDS: Final[float] = 6000.0
 # deadline and connect handshake (see ``http_handler`` cached handler paths).
 COMPLETION_HTTP_FALLBACK_SECONDS: Final[float] = 600.0
 HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: Final[float] = 5.0
+SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS: Final[float] = float(
+    os.getenv("SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS", "5.0")
+)
 request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS))))
 request_timeout_explicitly_set: bool = "REQUEST_TIMEOUT" in os.environ
 DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000))  # 10 minutes
@@ -467,6 +483,9 @@ MAX_TIME_TO_CLEAR_QUEUE: Final = float(os.getenv("MAX_TIME_TO_CLEAR_QUEUE", 5.0)
 LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS: Final = float(
     os.getenv("LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS", 0.5)
 )  # Cooldown time in seconds before allowing another aggressive clear (default: 0.5s)
+LOGGING_EXECUTOR_MAX_THREADS: Final = get_env_int("LOGGING_EXECUTOR_MAX_THREADS", 100)
+LOGGING_EXECUTOR_MAX_PENDING_TASKS: Final = get_env_int("LOGGING_EXECUTOR_MAX_PENDING_TASKS", 10_000)
+LOGGING_EXECUTOR_DROPPED_TASK_LOG_INTERVAL_SECONDS: Final = 30.0
 DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE: Final = os.getenv(
     "DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE", "streaming.chunk.yield"
 )
@@ -763,6 +782,7 @@ openai_compatible_endpoints: Final[list] = [
     "https://api.libertai.io/v1",
     "https://pinstripes.io/v1",
     "https://api.meta.ai/v1",
+    "https://api.cognition.ai/v1",
 ]
 
 
@@ -830,6 +850,7 @@ openai_compatible_providers: Final[list] = [
     "pinstripes",  # Pinstripes - JSON-configured provider
     "darkbloom",
     "meta",  # Meta Model API (Muse Spark) - JSON-configured provider
+    "cognition",
 ]
 openai_text_completion_compatible_providers: Final[list] = [  # providers that support `/v1/completions`
     "together_ai",
@@ -1342,6 +1363,11 @@ LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
     "Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.). "
     "To increase the truncation limit, set `MAX_STRING_LENGTH_PROMPT_IN_DB` in your env."
 )
+LITELLM_TRUNCATION_STDOUT_SAFEGUARD_NOTE: Final = (
+    "Truncation is a stdout logging safeguard. "
+    "Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.) and at DEBUG level. "
+    "To increase the truncation limit, set `MAX_STRING_LENGTH_STDOUT_LOG` in your env."
+)
 
 ########################### LiteLLM Proxy Specific Constants ###########################
 ########################################################################################
@@ -1516,6 +1542,11 @@ DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_
 PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597))
 RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_BATCH_SIZE", "500")))
 RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN", "100")))
+RESET_BUDGET_JOB_NAME: Final = "reset_budget_job"
+# Comfortably longer than one PROXY_BUDGET_RESCHEDULER_MIN_TIME tick, so a healthy
+# leader keeps the lease across its own run, and a crashed one strands the sweep for
+# at most a single tick.
+RESET_BUDGET_JOB_LOCK_TTL_SECONDS: Final[int] = 900
 PROXY_BATCH_POLLING_INTERVAL: Final = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 3600))
 MAX_OBJECTS_PER_POLL_CYCLE: Final = max(1, int(os.getenv("MAX_OBJECTS_PER_POLL_CYCLE", 50)))
 MANAGED_OBJECT_STALENESS_CUTOFF_DAYS: Final = max(1, int(os.getenv("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7)))
@@ -1580,6 +1611,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     "public_model_groups_links",
     "cost_discount_config",
     "cost_margin_config",
+    "block_requests_for_models_without_pricing",
     "budget_exceeded_throttle_percentage",
     # Every field editable from the Admin UI (proxy_server._GENERAL_SETTINGS_UI_LITELLM_FIELDS)
     # must be listed here so a DB write from one worker overrides the live litellm attribute on

litellm/exceptions.py

@@ -1039,16 +1039,29 @@ class LiteLLMUnknownProvider(BadRequestError):
 
 
 class GuardrailRaisedException(Exception):
+    """
+    Raised both when a guardrail judged content and when it could not judge it at all, since a
+    guardrail that fails closed refuses the request the same way a policy violation does.
+
+    ``blocked_content`` separates the two. Set it only where the guardrail actually reached a
+    verdict on the payload; leave it alone for an unreachable backend, a timeout, or a response
+    the integration could not parse. Callers that treat a block as something other than a plain
+    failure, such as the batch path dropping one record and submitting the rest, must gate on it,
+    because dropping a record no guardrail ever inspected is a silent loss of enforcement.
+    """
+
     def __init__(
         self,
         guardrail_name: str | None = None,
         message: str = "",
         should_wrap_with_default_message: bool = True,
         status_code: int = 400,
+        blocked_content: bool = False,
     ):
         default_message: Final = f"Guardrail raised an exception, Guardrail: {guardrail_name}, Message: {message}"
         self.guardrail_name = guardrail_name
         self.status_code = status_code
+        self.blocked_content = blocked_content
         self.message = default_message if should_wrap_with_default_message else message
         super().__init__(self.message)
 

litellm/experimental_mcp_client/client.py

@@ -51,6 +51,42 @@ def to_basic_auth(auth_value: str) -> str:
     return base64.b64encode(auth_value.encode("utf-8")).decode()
 
 
+def strip_auth_scheme(auth_value: str, scheme: str) -> str:
+    """Return ``auth_value`` with a leading ``<scheme> `` removed, or unchanged when absent.
+
+    Callers supply both a bare credential and a complete header value, so prefixing
+    unconditionally yields ``Bearer Bearer <jwt>``. Scheme names are case-insensitive per
+    RFC 7235. A credential is required after the scheme, so both a token that merely begins
+    with the scheme text and a scheme with nothing behind it are returned untouched.
+    Surrounding whitespace is left to ``_strip_header_whitespace`` at header-build time.
+    """
+    scheme_name, _, remainder = auth_value.lstrip().partition(" ")
+    credential: Final = remainder.lstrip()
+    if credential and scheme_name.lower() == scheme.lower():
+        return credential
+    return auth_value
+
+
+def to_basic_credentials(auth_value: str) -> str:
+    """Return the base64 credentials for a ``Basic`` header, encoding only when needed.
+
+    ``Basic <credentials>`` carries credentials that are already encoded, so encoding the whole
+    value again would bury the scheme inside the payload. This has to run before
+    :func:`to_basic_auth` rather than at header-build time, where no prefix is left to find.
+    A schemed value whose remainder does not decode is the bare ``username:password`` shape with
+    the scheme written in front of it, and is encoded rather than forwarded as an invalid header;
+    a pair always contains ``:``, which is outside the base64 alphabet, so the two never collide.
+    """
+    credentials: Final = strip_auth_scheme(auth_value, "Basic")
+    if credentials == auth_value:
+        return to_basic_auth(auth_value)
+    try:
+        base64.b64decode(credentials, validate=True)
+    except ValueError:
+        return to_basic_auth(credentials)
+    return credentials
+
+
 def _strip_header_whitespace(headers: dict[str, str]) -> dict[str, str]:
     return {
         (key.strip() if isinstance(key, str) else key): (value.strip() if isinstance(value, str) else value)
@@ -441,16 +477,15 @@ class MCPClient:
                 except BaseException as e:
                     verbose_logger.debug("Error during http_client cleanup: %s", e)
 
-    def update_auth_value(self, mcp_auth_value: str | dict[str, str]):
+    def update_auth_value(self, mcp_auth_value: str | dict[str, str]) -> None:
         """
         Set the authentication header for the MCP client.
         """
         if isinstance(mcp_auth_value, dict):
             self._mcp_auth_value = mcp_auth_value
+        elif self.auth_type == MCPAuth.basic:
+            self._mcp_auth_value = to_basic_credentials(mcp_auth_value)
         else:
-            if self.auth_type == MCPAuth.basic:
-                # Assuming mcp_auth_value is in format "username:password", convert it when updating
-                mcp_auth_value = to_basic_auth(mcp_auth_value)
             self._mcp_auth_value = mcp_auth_value
 
     def _get_auth_headers(self) -> dict:
@@ -459,19 +494,20 @@ class MCPClient:
         if self._mcp_auth_value:
             if isinstance(self._mcp_auth_value, str):
                 if self.auth_type == MCPAuth.bearer_token:
-                    headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
+                    headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
                 elif self.auth_type == MCPAuth.basic:
                     headers["Authorization"] = f"Basic {self._mcp_auth_value}"
                 elif self.auth_type == MCPAuth.api_key:
                     headers["X-API-Key"] = self._mcp_auth_value
                 elif self.auth_type == MCPAuth.authorization:
+                    # This auth type means the caller owns the whole header value.
                     headers["Authorization"] = self._mcp_auth_value
                 elif self.auth_type == MCPAuth.oauth2:
-                    headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
+                    headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
                 elif self.auth_type == MCPAuth.token:
-                    headers["Authorization"] = f"token {self._mcp_auth_value}"
+                    headers["Authorization"] = f"token {strip_auth_scheme(self._mcp_auth_value, 'token')}"
                 elif self.auth_type == MCPAuth.oauth2_token_exchange:
-                    headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
+                    headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
             elif isinstance(self._mcp_auth_value, dict):
                 headers.update(self._mcp_auth_value)
         # Note: aws_sigv4 auth is not handled here — SigV4 requires per-request

litellm/integrations/anthropic_cache_control_hook.py

@@ -27,11 +27,15 @@ from litellm.types.integrations.anthropic_cache_control_hook import (
     CacheControlInjectionPoint,
     CacheControlMessageInjectionPoint,
 )
-from litellm.types.llms.anthropic import AnthropicSystemMessageContent
+from litellm.types.llms.anthropic import (
+    AllAnthropicToolsValues,
+    AnthropicSystemMessageContent,
+)
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionCachedContent,
     ChatCompletionTextObject,
+    ChatCompletionToolParam,
     PromptCacheBreakpoint,
     PromptCacheOptions,
 )
@@ -57,6 +61,8 @@ OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
 OPENAI_API_HOST: Final = "api.openai.com"
 OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE")
 
+AllToolParamValues = ChatCompletionToolParam | AllAnthropicToolsValues
+
 
 def supports_openai_prompt_cache_breakpoint(model: str) -> bool:
     model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
@@ -625,6 +631,50 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         ]
         return points
 
+    @staticmethod
+    def messages_with_default_injections(
+        messages: list[AllMessageValues],
+        models: Iterable[str],
+        tools: list[AllToolParamValues] | None = None,
+        enable_prompt_caching: bool | None = None,
+    ) -> list[AllMessageValues]:
+        """Return the messages auto prompt caching will send, default breakpoints included.
+
+        Router cache affinity depends on this. Deployment selection runs before the injection in
+        `litellm.acompletion`, so it has to reproduce the markers to derive the same cache key the
+        success event later writes from the sent messages. `models` is every candidate model of the
+        group: the first that would auto-inject decides, since the default breakpoints (system
+        prompt and trailing turn) do not depend on which deployment serves the call. Returns the
+        input list itself when auto-injection would not apply
+        """
+        points: Final = next(
+            (
+                candidate
+                for candidate in (
+                    AnthropicCacheControlHook.get_default_injection_points(
+                        messages=messages,
+                        system=None,
+                        model=model,
+                        custom_llm_provider=None,
+                        tools=tools,
+                        enable_prompt_caching=enable_prompt_caching,
+                    )
+                    for model in models
+                )
+                if candidate
+            ),
+            None,
+        )
+        if not points:
+            return messages
+        return AnthropicCacheControlHook._apply_message_injections(
+            points=cast(  # cast-ok: the default points are all message-location points
+                list[CacheControlMessageInjectionPoint], points
+            ),
+            messages=copy.deepcopy(messages),
+            max_blocks=MAX_CACHE_CONTROL_BLOCKS,
+        )
+
     @staticmethod
     def maybe_seed_default_injection_points(
         non_default_params: dict[str, Any],

litellm/integrations/custom_guardrail.py

@@ -65,6 +65,41 @@ _guardrail_self_recorded: Final[contextvars.ContextVar[bool]] = contextvars.Cont
 )
 
 
+def is_guardrail_intervention(e: Exception) -> bool:
+    """
+    Returns True if the exception represents an intentional guardrail block
+    (this was logged previously as an API failure - guardrail_failed_to_respond).
+
+    Guardrails signal intentional blocks by raising:
+    - GuardrailRaisedException (generic guardrail API, tool permission)
+    - BlockedPiiEntityError (Presidio PII detection)
+    - SensitiveDataRouteException (sensitive-data reroute to on-premise model)
+    - HTTPException with a block-signalling status (400, 403, 422)
+    - ModifyResponseException (passthrough mode violation)
+
+    Only the statuses guardrails use in-tree to signal a deliberate rejection
+    count as an intervention: 400 (content policy), 403 (e.g. akto) and 422
+    (e.g. llm_as_a_judge). Other 4xx codes are commonly propagated from an
+    upstream guardrail provider response (401 bad key, 408 timeout, 429 rate
+    limit, or a raw upstream status), which are technical failures, not
+    blocks, so they stay guardrail_failed_to_respond.
+    """
+    if isinstance(e, ModifyResponseException):
+        return True
+    if isinstance(
+        e,
+        (
+            GuardrailRaisedException,
+            BlockedPiiEntityError,
+            SensitiveDataRouteException,
+        ),
+    ):
+        return True
+    if HTTPException is not None and isinstance(e, HTTPException) and e.status_code in _GUARDRAIL_BLOCK_STATUS_CODES:
+        return True
+    return False
+
+
 def _strict_guardrail_modes_enabled() -> bool:
     """Whether guardrail-mode validation raises (default) or logs a warning.
 
@@ -429,11 +464,13 @@ class CustomGuardrail(CustomLogger):
                         f"Sensitive data detected by {self.guardrail_name} (routing skipped: request has no session_id)"
                     ),
                     guardrail_name=self.guardrail_name,
+                    blocked_content=True,
                 )
         else:
             raise GuardrailRaisedException(
                 message=f"Sensitive data detected by {self.guardrail_name}",
                 guardrail_name=self.guardrail_name,
+                blocked_content=True,
             )
 
     @staticmethod
@@ -1068,42 +1105,8 @@ class CustomGuardrail(CustomLogger):
 
     @staticmethod
     def _is_guardrail_intervention(e: Exception) -> bool:
-        """
-        Returns True if the exception represents an intentional guardrail block
-        (this was logged previously as an API failure - guardrail_failed_to_respond).
-
-        Guardrails signal intentional blocks by raising:
-        - GuardrailRaisedException (generic guardrail API, tool permission)
-        - BlockedPiiEntityError (Presidio PII detection)
-        - SensitiveDataRouteException (sensitive-data reroute to on-premise model)
-        - HTTPException with a block-signalling status (400, 403, 422)
-        - ModifyResponseException (passthrough mode violation)
-
-        Only the statuses guardrails use in-tree to signal a deliberate rejection
-        count as an intervention: 400 (content policy), 403 (e.g. akto) and 422
-        (e.g. llm_as_a_judge). Other 4xx codes are commonly propagated from an
-        upstream guardrail provider response (401 bad key, 408 timeout, 429 rate
-        limit, or a raw upstream status), which are technical failures, not
-        blocks, so they stay guardrail_failed_to_respond.
-        """
-        if isinstance(e, ModifyResponseException):
-            return True
-        if isinstance(
-            e,
-            (
-                GuardrailRaisedException,
-                BlockedPiiEntityError,
-                SensitiveDataRouteException,
-            ),
-        ):
-            return True
-        if (
-            HTTPException is not None
-            and isinstance(e, HTTPException)
-            and e.status_code in _GUARDRAIL_BLOCK_STATUS_CODES
-        ):
-            return True
-        return False
+        """Retained spelling for existing callers; prefer ``is_guardrail_intervention``."""
+        return is_guardrail_intervention(e)
 
     def _process_error(
         self,

litellm/integrations/datadog/datadog_cost_management.py

@@ -11,6 +11,7 @@ from litellm.integrations.datadog.datadog_handler import (
     get_datadog_hostname,
     get_datadog_pod_name,
     get_datadog_service,
+    normalize_datadog_tag_value,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.custom_httpx.http_handler import (
@@ -184,7 +185,7 @@ class DatadogCostManagementLogger(CustomBatchLogger):
 
         # Backwards-compat: team/user/model_group preserved regardless of allowlist.
         if metadata.get("user_api_key_alias"):
-            tags["user"] = str(metadata["user_api_key_alias"])
+            tags["user"] = normalize_datadog_tag_value(metadata["user_api_key_alias"])
         team_tag: Final = (
             metadata.get("user_api_key_team_alias")
             or metadata.get("team_alias")
@@ -192,7 +193,7 @@ class DatadogCostManagementLogger(CustomBatchLogger):
             or metadata.get("team_id")
         )
         if team_tag:
-            tags["team"] = str(team_tag)
+            tags["team"] = normalize_datadog_tag_value(team_tag)
         if metadata.get("model_group"):
             tags["model_group"] = str(metadata["model_group"])
 
@@ -229,7 +230,7 @@ class DatadogCostManagementLogger(CustomBatchLogger):
                 value,
             )
             return
-        tags[key] = value
+        tags[key] = normalize_datadog_tag_value(value)
 
     @staticmethod
     def _add_tag(tags: dict[str, str], key: str, value: Any) -> None:

litellm/integrations/datadog/datadog_handler.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import re
 from typing import Final
 
 from litellm.types.utils import StandardLoggingPayload
@@ -36,6 +37,13 @@ def get_datadog_pod_name() -> str:
     return os.getenv("POD_NAME", "unknown")
 
 
+def normalize_datadog_tag_value(value: object) -> str:
+    normalized_value: Final = "".join(
+        character if character.isalnum() or character in "_-:./" else "_" for character in str(value).lower()
+    )
+    return re.sub(r"_+", "_", normalized_value).strip("_")
+
+
 def get_datadog_tags(
     standard_logging_object: StandardLoggingPayload | None = None,
 ) -> list[str]:
@@ -58,7 +66,7 @@ def get_datadog_tags(
 
     if standard_logging_object:
         request_tags: Final = standard_logging_object.get("request_tags", []) or []
-        tags.extend(f"request_tag:{tag}" for tag in request_tags)
+        tags.extend(f"request_tag:{normalize_datadog_tag_value(tag)}" for tag in request_tags)
 
         # Add Team Tag
         metadata: Final = standard_logging_object.get("metadata", {}) or {}
@@ -69,6 +77,6 @@ def get_datadog_tags(
             or metadata.get("team_id")
         )
         if team_tag:
-            tags.append(f"team:{team_tag}")
+            tags.append(f"team:{normalize_datadog_tag_value(team_tag)}")
 
     return tags

litellm/integrations/datadog/datadog_metrics.py

@@ -12,6 +12,7 @@ from litellm.integrations.datadog.datadog_handler import (
     get_datadog_hostname,
     get_datadog_pod_name,
     get_datadog_service,
+    normalize_datadog_tag_value,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.custom_httpx.http_handler import (
@@ -97,7 +98,7 @@ class DatadogMetricsLogger(CustomBatchLogger):
         )
 
         if team_tag:
-            tags.append(f"team:{team_tag}")
+            tags.append(f"team:{normalize_datadog_tag_value(team_tag)}")
 
         return tags
 

litellm/integrations/otel/logger.py

@@ -9,7 +9,15 @@ from typing import TYPE_CHECKING, Any, Final, cast
 from opentelemetry.context import Context, attach, get_current
 from opentelemetry.sdk._logs import LoggerProvider
 from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.trace import Span, Tracer, get_current_span, use_span
+from opentelemetry.trace import (
+    INVALID_SPAN,
+    Link,
+    Span,
+    Tracer,
+    get_current_span,
+    set_span_in_context,
+    use_span,
+)
 
 import litellm
 from litellm._logging import verbose_logger
@@ -21,6 +29,7 @@ from litellm.integrations.otel.model.config import OpenTelemetryV2Config
 from litellm.integrations.otel.model.metadata import (
     LLMCallEvent,
     RequestIdentity,
+    auth_metadata,
     model_from_request_data,
 )
 from litellm.integrations.otel.model.payloads import (
@@ -118,6 +127,12 @@ _OTEL_MODULES: Final = (
 _OPEN_CALLS_MAX: Final = 10_000
 
 
+def _request_trace_links(context: Context | None) -> tuple[Link, ...] | None:
+    """A link back to the request trace, for a span detached into its own trace."""
+    anchor: Final = get_current_span(context).get_span_context()
+    return (Link(anchor),) if anchor.is_valid else None
+
+
 class _LLMCallSpan:
     """The state carried from the ``pre_call`` boundary to span close.
 
@@ -127,13 +142,24 @@ class _LLMCallSpan:
     own (worker-copied) ambient context using ``start_time_ns``. The presence of
     a carrier for a call at all is the proof that ``pre_call`` ran, i.e. that an
     upstream call was actually attempted.
+
+    ``provider`` is the routed provider the live span was opened on (``None`` on
+    the default route or when creation was deferred). It is held in the tenant
+    cache while the span is open so LRU eviction can't shut the provider down
+    under it, and must be released exactly once when the carrier is removed.
     """
 
-    __slots__ = ("span", "start_time_ns")
+    __slots__ = ("provider", "span", "start_time_ns")
 
-    def __init__(self, span: "Span | None", start_time_ns: int | None) -> None:
+    def __init__(
+        self,
+        span: "Span | None",
+        start_time_ns: int | None,
+        provider: "TracerProvider | None" = None,
+    ) -> None:
         self.span = span
         self.start_time_ns = start_time_ns
+        self.provider = provider
 
 
 class OpenTelemetryV2(CustomLogger):
@@ -258,26 +284,37 @@ class OpenTelemetryV2(CustomLogger):
         if call_id in self._open_llm_calls:
             return
         start_time_ns: Final = to_ns(datetime.now())
-        span: Span | None = None
         # Parent to the request's anchored root span (stable across the request),
         # falling back to ambient on the SDK path. Open the span live only when
         # that resolves to a recordable parent; otherwise defer to the close
         # callback (the thread-pool case, where the anchor isn't visible here).
+        # Do not route on the deferred path: creating or LRU-touching a tenant
+        # provider here would evict idle ones even though close re-routes.
         parent_context: Final = resolve_request_span_context()
-        if is_recordable_span(get_current_span(parent_context)):
-            span = self._emitter.start_span(
+        if not is_recordable_span(get_current_span(parent_context)):
+            self._store_open_call(call_id, _LLMCallSpan(span=None, start_time_ns=start_time_ns))
+            return
+        # A detached route roots its own trace instead (linked to the request
+        # trace) — see ``TenantRoute.detached``.
+        route: Final = self._tenant_tracers.route_for(self.tracer, call.dynamic_params, call.auth_metadata)
+        try:
+            span: Final = self._emitter.start_span(
                 SpanRole.LLM_CALL,
                 call.provisional_span_name,
-                parent_context=parent_context,
+                parent_context=(
+                    set_span_in_context(INVALID_SPAN, parent_context) if route.detached else parent_context
+                ),
                 start_time_ns=start_time_ns,
-                tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params),
+                tracer=route.tracer,
+                links=_request_trace_links(parent_context) if route.detached else None,
             )
-        self._open_llm_calls[call_id] = _LLMCallSpan(span=span, start_time_ns=start_time_ns)
-        # Evict the oldest open call if the map is over budget. A call that opens
-        # but never closes (a stream that only fires stream events) would linger
-        # otherwise; the evicted span is simply dropped (never exported).
-        if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
-            self._open_llm_calls.popitem(last=False)
+        except BaseException:
+            self._tenant_tracers.release(route.provider)
+            raise
+        self._store_open_call(
+            call_id,
+            _LLMCallSpan(span=span, start_time_ns=start_time_ns, provider=route.provider),
+        )
 
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         if self._emit_mcp_tool_call(kwargs, start_time, end_time):
@@ -371,17 +408,22 @@ class OpenTelemetryV2(CustomLogger):
         # otherwise linger until evicted; drop it so it's neither leaked nor closed
         # as a phantom LLM span.
         if data.identity.call_id:
-            self._open_llm_calls.pop(data.identity.call_id, None)
-        parent_context, links = resolve_mcp_span_context()
-        parent_context = self._seed_identity_baggage(data.identity, None, parent_context)
-        self._emitter.emit(
-            SpanRole.MCP_TOOL_CALL,
-            data,
-            parent_context=parent_context,
-            start_time_ns=to_ns(start_time),
-            end_time_ns=to_ns(end_time),
-            links=links,
-        )
+            self._release_carrier(self._open_llm_calls.pop(data.identity.call_id, None))
+        route: Final = self._tenant_tracers.route_for(self.tracer, None, auth_metadata(payload, kwargs))
+        try:
+            parent_context, links = resolve_mcp_span_context()
+            seeded: Final = self._seed_identity_baggage(data.identity, None, parent_context)
+            self._emitter.emit(
+                SpanRole.MCP_TOOL_CALL,
+                data,
+                parent_context=(set_span_in_context(INVALID_SPAN, seeded) if route.detached else seeded),
+                start_time_ns=to_ns(start_time),
+                end_time_ns=to_ns(end_time),
+                links=((*(links or ()), *(_request_trace_links(seeded) or ())) if route.detached else links),
+                tracer=route.tracer,
+            )
+        finally:
+            self._tenant_tracers.release(route.provider)
         return True
 
     def _emit_mcp_list_tools(
@@ -407,17 +449,22 @@ class OpenTelemetryV2(CustomLogger):
             payload, capture_content=self.config.capture_span_content
         )
         if data.identity.call_id:
-            self._open_llm_calls.pop(data.identity.call_id, None)
-        parent_context, links = resolve_mcp_span_context()
-        parent_context = self._seed_identity_baggage(data.identity, None, parent_context)
-        self._emitter.emit(
-            SpanRole.MCP_LIST_TOOLS,
-            data,
-            parent_context=parent_context,
-            start_time_ns=to_ns(start_time),
-            end_time_ns=to_ns(end_time),
-            links=links,
-        )
+            self._release_carrier(self._open_llm_calls.pop(data.identity.call_id, None))
+        route: Final = self._tenant_tracers.route_for(self.tracer, None, auth_metadata(payload, kwargs))
+        try:
+            parent_context, links = resolve_mcp_span_context()
+            seeded: Final = self._seed_identity_baggage(data.identity, None, parent_context)
+            self._emitter.emit(
+                SpanRole.MCP_LIST_TOOLS,
+                data,
+                parent_context=(set_span_in_context(INVALID_SPAN, seeded) if route.detached else seeded),
+                start_time_ns=to_ns(start_time),
+                end_time_ns=to_ns(end_time),
+                links=((*(links or ()), *(_request_trace_links(seeded) or ())) if route.detached else links),
+                tracer=route.tracer,
+            )
+        finally:
+            self._tenant_tracers.release(route.provider)
         return True
 
     def _close_llm_call(
@@ -439,6 +486,36 @@ class OpenTelemetryV2(CustomLogger):
         carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None
         if carrier is None:
             return None
+        try:
+            return self._finish_carrier(carrier, call, end_time)
+        finally:
+            # After the span has ended, so a release-triggered provider shutdown
+            # force-flushes it out rather than racing its enqueue.
+            self._release_carrier(carrier)
+
+    def _store_open_call(self, call_id: str, carrier: _LLMCallSpan) -> None:
+        """Remember an in-flight LLM call, evicting the oldest if over budget.
+
+        A call that opens but never closes (a stream that only fires stream
+        events) would linger otherwise; the evicted span is simply dropped
+        (never exported).
+        """
+        self._open_llm_calls[call_id] = carrier
+        if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
+            _, evicted = self._open_llm_calls.popitem(last=False)
+            self._release_carrier(evicted)
+
+    def _release_carrier(self, carrier: "_LLMCallSpan | None") -> None:
+        """Release the routed provider a removed carrier was holding open."""
+        if carrier is not None:
+            self._tenant_tracers.release(carrier.provider)
+
+    def _finish_carrier(
+        self,
+        carrier: _LLMCallSpan,
+        call: LLMCallEvent,
+        end_time: datetime | float | None,
+    ) -> Span | None:
         payload: Final = call.payload
         if payload is None:
             if carrier.span is not None:
@@ -462,16 +539,23 @@ class OpenTelemetryV2(CustomLogger):
         # The worker copied the request task's context, which carries the anchored
         # root span — parent to it (ambient fallback on the SDK path). Seed identity
         # Baggage so the span — and the SDK path, which has none — is labeled
-        # consistently.
-        parent_ctx = self._seed_identity_baggage(data.identity, data.request_model, resolve_request_span_context())
-        return self._emitter.emit(
-            SpanRole.LLM_CALL,
-            data,
-            parent_context=parent_ctx,
-            start_time_ns=carrier.start_time_ns,
-            end_time_ns=end_time_ns,
-            tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params),
-        )
+        # consistently. A detached route roots its own trace instead, linked back.
+        route: Final = self._tenant_tracers.route_for(self.tracer, call.dynamic_params, call.auth_metadata)
+        try:
+            parent_ctx: Final = self._seed_identity_baggage(
+                data.identity, data.request_model, resolve_request_span_context()
+            )
+            return self._emitter.emit(
+                SpanRole.LLM_CALL,
+                data,
+                parent_context=(set_span_in_context(INVALID_SPAN, parent_ctx) if route.detached else parent_ctx),
+                start_time_ns=carrier.start_time_ns,
+                end_time_ns=end_time_ns,
+                tracer=route.tracer,
+                links=_request_trace_links(parent_ctx) if route.detached else None,
+            )
+        finally:
+            self._tenant_tracers.release(route.provider)
 
     # ====================================================================== #
     #  Service hooks

litellm/integrations/otel/model/metadata.py

@@ -36,8 +36,9 @@ model. They coincide on the SDK path, which is correct.
 
 from __future__ import annotations
 
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
 from dataclasses import dataclass, field
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, cast
 
 from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
@@ -195,6 +196,10 @@ class LLMCallEvent:
     # The ``standard_callback_dynamic_params`` routing the call to a per-tenant
     # tracer (its own exporter/endpoint), or ``None`` when the call isn't scoped.
     dynamic_params: Any
+    # The key/team config the proxy resolved at auth (``user_api_key_auth_metadata``),
+    # routing the call to that tenant's telemetry project. Server-set and so
+    # trusted, unlike ``dynamic_params``, which carries client-supplied metadata.
+    auth_metadata: Mapping[str, str] | None
     # True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire
     # the ``pre_call`` hook but never made an upstream call, so they get no span.
     is_no_upstream_call: bool
@@ -214,6 +219,7 @@ class LLMCallEvent:
             call_id=_call_id(payload, kwargs),
             payload=payload,
             dynamic_params=kwargs.get("standard_callback_dynamic_params"),
+            auth_metadata=auth_metadata(payload, kwargs),
             is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
             provisional_span_name=f"{operation.value} {model}".strip(),
             time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
@@ -235,6 +241,64 @@ def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
     return completion_start - api_call_start
 
 
+def auth_metadata(payload: StandardLoggingPayload | None, kwargs: Mapping[str, object]) -> Mapping[str, str] | None:
+    """The key/team config the proxy resolved at auth, or ``None`` off the proxy.
+
+    Read from the payload once the call closes and from ``litellm_params`` at
+    ``pre_call``, where no payload exists yet — the LLM-call span is *created* at
+    ``pre_call``, so the tracer (and therefore the destination) must be
+    resolvable there. Values arrive untyped, so non-string entries are dropped
+    rather than passed on to header builders.
+    """
+    return next(
+        (
+            typed
+            for metadata in _metadata_dicts(payload, kwargs)
+            if (typed := _string_entries(metadata.get("user_api_key_auth_metadata")))
+        ),
+        None,
+    )
+
+
+def _as_str_mapping(value: object) -> Mapping[str, object] | None:
+    """A read-only view of ``value`` when it is a mapping, else ``None``."""
+    if not isinstance(value, Mapping):
+        return None
+    return cast("Mapping[str, object]", value)  # cast-ok: isinstance-guarded, JSON metadata has str keys
+
+
+def _string_entries(value: object) -> Mapping[str, str] | None:
+    entries: Final = _as_str_mapping(value)
+    if entries is None:
+        return None
+    typed: Final = MappingProxyType({key: item for key, item in entries.items() if isinstance(item, str)})
+    return typed or None
+
+
+def _metadata_dicts(
+    payload: StandardLoggingPayload | None, kwargs: Mapping[str, object]
+) -> Iterator[Mapping[str, object]]:
+    """Request metadata dicts, closed-call payload first then the live kwargs.
+
+    ``litellm_metadata`` is the metadata field on the Anthropic-shaped routes;
+    litellm copies it onto ``metadata``, but both are yielded so a route that
+    populates only one is still covered.
+    """
+    payload_view: Final = _as_str_mapping(payload)
+    if payload_view is not None:
+        payload_metadata: Final = _as_str_mapping(payload_view.get("metadata"))
+        if payload_metadata is not None:
+            yield payload_metadata
+    params: Final = _as_str_mapping(kwargs.get("litellm_params"))
+    if params is None:
+        return
+    yield from (
+        metadata
+        for key in ("metadata", "litellm_metadata")
+        if (metadata := _as_str_mapping(params.get(key))) is not None
+    )
+
+
 def _call_id(payload: StandardLoggingPayload | None, kwargs: Mapping[str, Any]) -> str | None:
     """The call id from the payload (when closed) or the bare kwargs (at pre_call)."""
     if payload is not None:

litellm/integrations/otel/plumbing/routing.py

@@ -1,31 +1,45 @@
 """Per-request multi-tenant tracer routing.
 
 When a request carries team/key vendor credentials in
-``standard_callback_dynamic_params``, its spans must export through a
-``TracerProvider`` whose OTLP headers carry those credentials.
-``TenantTracerCache`` builds and caches one provider per distinct credential
-set, and otherwise hands back the logger's default tracer. This lets a single
-logger fan requests out to many tenants without needing a logger per tenant.
+``standard_callback_dynamic_params``, or the key/team config resolved at auth
+names a destination project, its spans must export through a
+``TracerProvider`` whose OTLP headers carry those credentials / that project.
+``TenantTracerCache`` builds and caches one provider per distinct
+(credentials, project) pair, and otherwise hands back the logger's default
+tracer. This lets a single logger fan requests out to many tenants without
+needing a logger per tenant.
 """
 
+import threading
 from collections import OrderedDict
 from collections.abc import Mapping
-from typing import Any, Final
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Any, Final, TypeAlias
+from urllib.parse import quote
 
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.trace import Tracer
 
 from litellm._logging import verbose_logger
-from litellm.integrations.otel.model.config import OpenTelemetryV2Config
+from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
 from litellm.integrations.otel.plumbing.providers import (
     build_tracer_provider,
     get_tracer,
 )
-from litellm.integrations.otel.presets import dynamic_otlp_headers
+from litellm.integrations.otel.presets import (
+    dynamic_otlp_headers,
+    project_routing_headers,
+)
 
 # Exporter kinds that ignore headers — never rewritten with dynamic credentials.
 _NON_OTLP_KINDS: Final = ("console", "in_memory", "inmemory", "memory")
 
+# gRPC exporters still take dynamic credentials (as gRPC metadata) but not
+# project headers: the routing headers backends read (Phoenix's
+# ``x-project-name``) are only honored on the OTLP/HTTP endpoint.
+_GRPC_KINDS: Final = ("otlp_grpc", "grpc")
+
 # Cap on distinct credential-scoped providers held at once. ``dynamic_params``
 # can be populated from request metadata, so an unbounded cache lets a caller
 # spawn one ``TracerProvider`` (plus its ``BatchSpanProcessor`` background
@@ -34,6 +48,23 @@ _NON_OTLP_KINDS: Final = ("console", "in_memory", "inmemory", "memory")
 # evicted providers so their threads are reclaimed.
 _MAX_CACHED_PROVIDERS: Final = 256
 
+# Cap on providers evicted from the cache while still holding open spans, which
+# are kept alive to drain instead of being shut down under them. Their only
+# other bound is the logger's open-call map (10k), so without this a caller
+# cycling unique credential sets across long-lived calls could pin far more
+# live providers, and exporter threads, than the cache cap allows. Past this
+# many, the stalest retiree is shut down and whatever it was draining is
+# dropped (a shut-down ``BatchSpanProcessor`` discards spans handed to it after
+# the fact), which by then means a span on a route evicted long ago. A quarter
+# of the cache cap: enough that a burst of tenant churn during long-lived calls
+# still drains normally, small enough that the worst case is a bounded 320
+# providers rather than one per concurrent call.
+_MAX_RETIRED_PROVIDERS: Final = 64
+
+_HeaderItems: TypeAlias = tuple[tuple[str, str], ...]
+
+_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
+
 
 def _shutdown_provider(provider: TracerProvider) -> None:
     """Flush + stop an evicted provider's processors (reclaims their threads).
@@ -49,8 +80,41 @@ def _shutdown_provider(provider: TracerProvider) -> None:
         verbose_logger.debug("OTel V2: error shutting down evicted provider: %s", e)
 
 
+def _plain_header_string(headers: Mapping[str, str]) -> str:
+    return ",".join(f"{key}={value}" for key, value in headers.items())
+
+
+def _encoded_header_string(headers: Mapping[str, str]) -> str:
+    """Percent-encode values so one containing the ``k=v,k=v`` separators (e.g.
+    a project name with a comma) survives; ``parse_env_headers`` decodes it back.
+    """
+    return ",".join(f"{key}={quote(value, safe='')}" for key, value in headers.items())
+
+
+@dataclass(frozen=True, slots=True)
+class TenantRoute:
+    """The tracer to create a span on, plus whether it must root its own trace.
+
+    ``detached`` is True when project routing engaged. Phoenix assigns a whole
+    trace to one project by whichever of its spans arrives first, so a
+    project-routed span parented into the request trace gets dragged into the
+    project of the default-exported request spans and the header does nothing.
+    The span must therefore start a fresh trace (with a link back to the
+    request trace for correlation) — which is also how the v1 Phoenix logger
+    behaved, exporting each request under its own Phoenix-local parent span.
+    """
+
+    tracer: Tracer
+    detached: bool
+    #: The provider ``tracer`` came from, or ``None`` on the default route. It
+    #: is returned already held (counted as an open span, atomically with the
+    #: cache update), so LRU eviction can't shut it down before the caller's
+    #: span lands; the caller must ``release`` it exactly once when done.
+    provider: TracerProvider | None = None
+
+
 class TenantTracerCache:
-    """Credential-scoped ``TracerProvider`` cache keyed by the dynamic headers."""
+    """Credential/project-scoped ``TracerProvider`` cache keyed by the routing headers."""
 
     def __init__(
         self,
@@ -61,49 +125,170 @@ class TenantTracerCache:
         self._config = config
         self._callback_name = callback_name
         self._tracer_name = tracer_name
-        self._providers: OrderedDict[tuple[tuple[str, str], ...], TracerProvider] = OrderedDict()
+        # Guards the three mutable structures below: ``pre_call`` can run on
+        # thread-pool workers concurrently with the event loop, so cache
+        # updates, span counts, and retirement must be atomic.
+        self._lock: Final = threading.Lock()
+        self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems], TracerProvider] = OrderedDict()
+        self._open_span_counts: dict[TracerProvider, int] = {}  # mutable-ok: live refcount state
+        # Oldest-first so an overflow of draining providers sheds the stalest.
+        self._retired: OrderedDict[TracerProvider, None] = OrderedDict()  # mutable-ok: draining evicted providers
+        self._project_routable = any(
+            spec.owner == callback_name and spec.kind.lower() not in (*_NON_OTLP_KINDS, *_GRPC_KINDS)
+            for spec in config.exporters
+        )
+        self._warned_project_unroutable = False
 
-    def tracer_for(self, default: Tracer, dynamic_params: Any) -> Tracer:
-        """Return the tracer for this request.
+    def release(self, provider: TracerProvider | None) -> None:
+        """Drop one open-span count; shut a retired provider down once drained.
 
-        Use ``default`` unless the request's dynamic credentials require a
-        credential-scoped tracer, in which case build (or reuse) one. The cache
-        is a bounded LRU: the least-recently-used provider is flushed and shut
-        down on overflow so its exporter threads don't accumulate.
+        ``None`` (the default route) is a no-op so callers can release a
+        ``TenantRoute.provider`` unconditionally. The shutdown itself runs
+        outside the lock: it force-flushes over the network and must not stall
+        every concurrently routing request.
         """
-        headers: Final = dynamic_otlp_headers(self._callback_name, dynamic_params)
-        if not headers:
-            return default
-        cache_key: Final = tuple(sorted(headers.items()))
-        provider = self._providers.get(cache_key)
-        if provider is not None:
+        if provider is None:
+            return
+        with self._lock:
+            remaining: Final = self._open_span_counts.get(provider, 0) - 1
+            if remaining > 0:
+                self._open_span_counts[provider] = remaining
+                return
+            self._open_span_counts.pop(provider, None)
+            drained: Final = provider in self._retired
+            self._retired.pop(provider, None)
+        if drained:
+            _shutdown_provider(provider)
+
+    def route_for(
+        self,
+        default: Tracer,
+        dynamic_params: Any,
+        auth_metadata: Mapping[str, str] | None = None,
+    ) -> TenantRoute:
+        """Return the tracer (and trace-detachment flag) for this request.
+
+        Use ``default`` unless the request's dynamic credentials or its key/team
+        project require a scoped tracer, in which case build (or reuse) one. The
+        cache is a bounded LRU: the least-recently-used provider is flushed and
+        shut down on overflow so its exporter threads don't accumulate.
+
+        A routed provider is returned already held — its open-span count is
+        incremented in the same critical section as the cache update — so a
+        concurrent overflow eviction can't shut it down between selection and
+        the caller's span start. The caller must ``release`` it exactly once.
+        """
+        credential_headers: Final = dynamic_otlp_headers(self._callback_name, dynamic_params) or _NO_HEADERS
+        project_headers: Final = self._project_headers(auth_metadata)
+        if not credential_headers and not project_headers:
+            return TenantRoute(tracer=default, detached=False)
+        cache_key: Final = (
+            tuple(sorted(credential_headers.items())),
+            tuple(sorted(project_headers.items())),
+        )
+        with self._lock:
+            provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers)
+            self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1
+            evicted: Final = self._evicted_on_overflow_locked()
+        if evicted is not None:
+            _shutdown_provider(evicted)
+        return TenantRoute(
+            tracer=get_tracer(provider, self._tracer_name),
+            detached=bool(project_headers),
+            provider=provider,
+        )
+
+    def _cached_provider_locked(
+        self,
+        cache_key: tuple[_HeaderItems, _HeaderItems],
+        credential_headers: Mapping[str, str],
+        project_headers: Mapping[str, str],
+    ) -> TracerProvider:
+        cached: Final = self._providers.get(cache_key)
+        if cached is not None:
             self._providers.move_to_end(cache_key)
-        else:
-            provider = build_tracer_provider(self._config_with_headers(headers))
-            self._providers[cache_key] = provider
-            if len(self._providers) > _MAX_CACHED_PROVIDERS:
-                _, evicted = self._providers.popitem(last=False)
-                _shutdown_provider(evicted)
-        return get_tracer(provider, self._tracer_name)
+            return cached
+        built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers))
+        self._providers[cache_key] = built
+        return built
 
-    def _config_with_headers(self, headers: Mapping[str, str]) -> OpenTelemetryV2Config:
-        """Clone the config, stamping ``headers`` onto the credential's own exporter.
+    def _evicted_on_overflow_locked(self) -> TracerProvider | None:
+        """Pop the LRU provider past the cap; return it if the caller must shut it down.
 
-        ``headers`` are the per-request credentials of ``self._callback_name`` (the
-        integration that built this cache), so they apply only to the exporter that
-        integration contributed (``spec.owner``). A request that carries one
-        tenant's Arize key must never rewrite the headers of a co-configured
-        Langfuse or self-hosted collector exporter, which would leak that key to a
-        different backend.
+        A provider with open spans is retired to drain instead: stopping its
+        processors while a span opened at ``pre_call`` is still live would
+        silently drop that span at end instead of exporting it. Retirees are
+        themselves capped, so the stalest one is shut down (and its open-span
+        count dropped, making its eventual ``release`` a no-op) once too many
+        pile up rather than letting them accumulate a thread each.
         """
-        header_str: Final = ",".join(f"{key}={value}" for key, value in headers.items())
-        header_update: Final[dict[str, str]] = {"headers": header_str}
-        exporters: Final = [
-            (
-                spec.model_copy(update=header_update)
-                if spec.owner == self._callback_name and spec.kind.lower() not in _NON_OTLP_KINDS
-                else spec
+        if len(self._providers) <= _MAX_CACHED_PROVIDERS:
+            return None
+        _, evicted = self._providers.popitem(last=False)
+        if self._open_span_counts.get(evicted, 0) == 0:
+            return evicted
+        self._retired[evicted] = None
+        if len(self._retired) <= _MAX_RETIRED_PROVIDERS:
+            return None
+        overflowed, _ = self._retired.popitem(last=False)
+        self._open_span_counts.pop(overflowed, None)
+        return overflowed
+
+    def _project_headers(self, auth_metadata: Mapping[str, str] | None) -> Mapping[str, str]:
+        """The per-request project-routing headers, if this cache can apply them.
+
+        A gRPC-only exporter can't (the project header route is HTTP-only), so
+        the request warns once and stays on the env-configured default project.
+        """
+        requested: Final = project_routing_headers(self._callback_name, auth_metadata)
+        if not requested or self._project_routable:
+            return requested
+        if not self._warned_project_unroutable:
+            self._warned_project_unroutable = True
+            verbose_logger.warning(
+                "OTel V2: %s key/team config names a per-request project, but its exporter "
+                "is not OTLP/HTTP and the project header is HTTP-only; spans stay in the "
+                "default project.",
+                self._callback_name,
             )
-            for spec in self._config.exporters
+        return _NO_HEADERS
+
+    def _routed_config(
+        self,
+        credential_headers: Mapping[str, str],
+        project_headers: Mapping[str, str],
+    ) -> OpenTelemetryV2Config:
+        """Clone the config, rewriting headers on the callback's own exporter.
+
+        Both header sets apply only to the exporter ``self._callback_name``
+        contributed (``spec.owner``). A request that carries one tenant's Arize
+        key must never rewrite the headers of a co-configured Langfuse or
+        self-hosted collector exporter, which would leak that key to a
+        different backend.
+
+        Dynamic credentials REPLACE the exporter's headers — they are the
+        tenant's complete credential set. Project headers APPEND instead: the
+        preset's static headers carry the backend auth (Phoenix's
+        ``Authorization``), which must survive routing to a project.
+        """
+        exporters: Final = [
+            self._routed_exporter(spec, credential_headers, project_headers) for spec in self._config.exporters
         ]
         return self._config.model_copy(update={"exporters": exporters})
+
+    def _routed_exporter(
+        self,
+        spec: ExporterSpec,
+        credential_headers: Mapping[str, str],
+        project_headers: Mapping[str, str],
+    ) -> ExporterSpec:
+        kind: Final = spec.kind.lower()
+        if spec.owner != self._callback_name or kind in _NON_OTLP_KINDS:
+            return spec
+        base: Final = _plain_header_string(credential_headers) if credential_headers else spec.headers
+        routed: Final = (
+            ",".join(part for part in (base, _encoded_header_string(project_headers)) if part)
+            if project_headers and kind not in _GRPC_KINDS
+            else base
+        )
+        return spec if routed == spec.headers else spec.model_copy(update={"headers": routed})

litellm/integrations/otel/presets/__init__.py

@@ -8,7 +8,8 @@ the factory in ``litellm_logging`` can resolve a name and build a single
 ``OpenTelemetryV2`` instance from the result.
 """
 
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
+from types import MappingProxyType
 from typing import Final
 
 from litellm.integrations.otel.presets.agentops import agentops_preset
@@ -20,7 +21,10 @@ from litellm.integrations.otel.presets.langfuse import (
 )
 from litellm.integrations.otel.presets.langtrace import langtrace_preset
 from litellm.integrations.otel.presets.levo import levo_preset
-from litellm.integrations.otel.presets.phoenix import phoenix_preset
+from litellm.integrations.otel.presets.phoenix import (
+    phoenix_preset,
+    phoenix_project_headers,
+)
 from litellm.integrations.otel.presets.weave import weave_dynamic_headers, weave_preset
 from litellm.types.utils import StandardCallbackDynamicParams
 
@@ -47,6 +51,23 @@ DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicPa
 }
 
 
+#: Callback name → per-request *routing* header builder, sourced from the key/team
+#: config the proxy resolved at auth. Deliberately separate from
+#: ``DYNAMIC_HEADERS_BY_CALLBACK``: that one is fed
+#: ``StandardCallbackDynamicParams``, which is populated from client-supplied
+#: request metadata. Naming a destination project is a data-exfiltration
+#: primitive, so it must only ever come from server-set key/team config.
+PROJECT_HEADERS_BY_CALLBACK: Final[Mapping[str, Callable[[Mapping[str, str] | None], Mapping[str, str]]]] = (
+    MappingProxyType(
+        {
+            "arize_phoenix": phoenix_project_headers,
+        }
+    )
+)
+
+_NO_PROJECT_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
+
+
 def dynamic_otlp_headers(
     callback_name: str | None,
     dynamic_params: StandardCallbackDynamicParams | None,
@@ -62,9 +83,25 @@ def dynamic_otlp_headers(
     return headers or None
 
 
+def project_routing_headers(
+    callback_name: str | None,
+    auth_metadata: Mapping[str, str] | None,
+) -> Mapping[str, str]:
+    """Per-request project-routing headers from trusted key/team config.
+
+    Empty means "no per-request project" — the caller keeps its default tracer,
+    whose resource attributes carry the env-configured project.
+    """
+    builder: Final = PROJECT_HEADERS_BY_CALLBACK.get(callback_name or "")
+    if builder is None:
+        return _NO_PROJECT_HEADERS
+    return builder(auth_metadata)
+
+
 __all__ = [
     "DYNAMIC_HEADERS_BY_CALLBACK",
     "PRESET_BY_CALLBACK",
+    "PROJECT_HEADERS_BY_CALLBACK",
     "Preset",
     "agentops_preset",
     "arize_preset",
@@ -73,5 +110,6 @@ __all__ = [
     "langtrace_preset",
     "levo_preset",
     "phoenix_preset",
+    "project_routing_headers",
     "weave_preset",
 ]

litellm/integrations/otel/presets/phoenix.py

@@ -1,5 +1,7 @@
 """Arize-Phoenix preset."""
 
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import Final
 
 from pydantic import AliasChoices, Field
@@ -25,6 +27,36 @@ class _PhoenixSettings(BaseSettings):
     )
 
 
+#: Phoenix routes an OTLP/HTTP export to a project by this header, which takes
+#: precedence over the ``openinference.project.name`` resource attribute the env
+#: var sets. Requires arize-phoenix 15.5.0+; older collectors ignore it and the
+#: spans land in the resource attribute's project.
+PHOENIX_PROJECT_HEADER: Final = "x-project-name"
+
+#: Key/team config fields naming the target project, highest precedence first.
+_PROJECT_KEYS: Final = ("phoenix_project_name_override", "phoenix_project_name")
+
+_NO_PROJECT: Final[Mapping[str, str]] = MappingProxyType({})
+
+
+def phoenix_project_headers(auth_metadata: Mapping[str, str] | None) -> Mapping[str, str]:
+    """The per-request Phoenix project header for this key/team, if any.
+
+    ``auth_metadata`` must be the key/team config the proxy resolved at auth
+    (``user_api_key_auth_metadata``), never client-supplied request metadata:
+    choosing the destination project is a data-exfiltration primitive, so a
+    caller must not be able to name one. Returns an empty mapping when the key
+    and team name no project, leaving the request on the env-configured default.
+    """
+    if not auth_metadata:
+        return _NO_PROJECT
+    project: Final = next(
+        (stripped for key in _PROJECT_KEYS if (stripped := (auth_metadata.get(key) or "").strip())),
+        "",
+    )
+    return MappingProxyType({PHOENIX_PROJECT_HEADER: project}) if project else _NO_PROJECT
+
+
 def phoenix_preset(
     *,
     config_overrides: OpenTelemetryV2Config | None = None,

litellm/integrations/prometheus.py

@@ -4067,9 +4067,10 @@ class PrometheusLogger(CustomLogger):
             require_auth (bool, optional): Whether to require authentication for the metrics endpoint.
                                         Defaults to False.
         """
-        from prometheus_client import make_asgi_app
+        from prometheus_client import REGISTRY
 
         from litellm._logging import verbose_proxy_logger
+        from litellm.integrations.prometheus_metrics_endpoint import make_metrics_asgi_app
         from litellm.proxy.proxy_server import app
 
         # Create metrics ASGI app
@@ -4078,9 +4079,9 @@ class PrometheusLogger(CustomLogger):
 
             registry: Final = CollectorRegistry()
             multiprocess.MultiProcessCollector(registry)
-            metrics_app = make_asgi_app(registry)
+            metrics_app = make_metrics_asgi_app(registry)
         else:
-            metrics_app = make_asgi_app()
+            metrics_app = make_metrics_asgi_app(REGISTRY)
 
         # Mount the metrics app to the app
         app.mount("/metrics", metrics_app)

litellm/integrations/prometheus_metrics_endpoint.py

@@ -0,0 +1,100 @@
+"""ASGI app for `/metrics` that keeps registry rendering off the event loop.
+
+``prometheus_client.make_asgi_app`` collects and serializes the whole registry
+inline in the coroutine, so a large scrape (tens of MB on high cardinality
+deployments) blocks every other request on the loop for its whole duration. This
+app renders in a worker thread instead, shares one render across concurrent
+scrapes that want the same output, and streams the payload back in chunks.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import gzip
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final
+
+from prometheus_client import CollectorRegistry
+from prometheus_client.exposition import choose_encoder, gzip_accepted
+from starlette.requests import Request
+from starlette.responses import StreamingResponse
+from starlette.types import ASGIApp, Receive, Scope, Send
+
+RESPONSE_CHUNK_SIZE_BYTES: Final = 64 * 1024
+
+_GZIP_HEADERS: Final = MappingProxyType({"Content-Encoding": "gzip"})
+
+
+@dataclass(frozen=True, slots=True)
+class ScrapeRequest:
+    """What a scrape asks for, normalized so that header spellings sharing an output share a render."""
+
+    encoder: Callable[[CollectorRegistry], bytes]
+    content_type: str
+    gzipped: bool
+    metric_names: tuple[str, ...]
+
+
+def parse_scrape_request(accept: str, accept_encoding: str, metric_names: tuple[str, ...]) -> ScrapeRequest:
+    encoder, content_type = choose_encoder(accept)
+    return ScrapeRequest(
+        encoder=encoder,
+        content_type=content_type,
+        gzipped=gzip_accepted(accept_encoding),
+        metric_names=metric_names,
+    )
+
+
+def render_scrape(registry: CollectorRegistry, request: ScrapeRequest) -> bytes:
+    rendered: Final = request.encoder(
+        registry.restricted_registry(request.metric_names) if request.metric_names else registry  # pyright: ignore[reportArgumentType]  # RestrictedRegistry is registry-shaped but not a subclass
+    )
+    return gzip.compress(rendered) if request.gzipped else rendered
+
+
+class CoalescedScrapeRenderer:
+    """Renders the registry in a worker thread, sharing one render per distinct output across concurrent scrapes."""
+
+    def __init__(self, registry: CollectorRegistry) -> None:
+        self._registry = registry
+        self._inflight: Mapping[ScrapeRequest, asyncio.Task[bytes]] = MappingProxyType({})
+
+    def _forget(self, finished: asyncio.Task[bytes]) -> None:
+        self._inflight = MappingProxyType({key: task for key, task in self._inflight.items() if task is not finished})
+
+    async def render(self, request: ScrapeRequest) -> bytes:
+        inflight: Final = self._inflight.get(request)
+        if inflight is not None:
+            return await asyncio.shield(inflight)
+
+        task: Final = asyncio.create_task(asyncio.to_thread(render_scrape, self._registry, request))
+        self._inflight = MappingProxyType({**self._inflight, request: task})
+        task.add_done_callback(self._forget)
+        return await asyncio.shield(task)
+
+
+def _chunks(body: bytes) -> Iterator[bytes]:
+    return (body[start : start + RESPONSE_CHUNK_SIZE_BYTES] for start in range(0, len(body), RESPONSE_CHUNK_SIZE_BYTES))
+
+
+def make_metrics_asgi_app(registry: CollectorRegistry) -> ASGIApp:
+    renderer: Final = CoalescedScrapeRenderer(registry)
+
+    async def metrics_app(scope: Scope, receive: Receive, send: Send) -> None:
+        request: Final = Request(scope, receive)
+        scrape: Final = parse_scrape_request(
+            accept=request.headers.get("accept", ""),
+            accept_encoding=request.headers.get("accept-encoding", ""),
+            metric_names=tuple(request.query_params.getlist("name[]")),
+        )
+        body: Final = await renderer.render(scrape)
+        response: Final = StreamingResponse(
+            _chunks(body),
+            media_type=scrape.content_type,
+            headers=_GZIP_HEADERS if scrape.gzipped else None,
+        )
+        await response(scope, receive, send)
+
+    return metrics_app

litellm/integrations/shadow_eval_logger.py

@@ -10,7 +10,7 @@ import asyncio
 import hashlib
 import random
 import traceback
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Awaitable, Callable, Mapping, Sequence
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from itertools import groupby
@@ -42,8 +42,9 @@ if TYPE_CHECKING:
     from litellm.router import Router
     from litellm.types.utils import StandardLoggingPayload
 
-# A job starting, stopping, or hitting its turn budget propagates to sampling within one
-# TTL; the turn budget can overshoot by at most one TTL of in-flight samples per pod.
+# A job starting, stopping, or hitting a budget propagates to sampling within one TTL;
+# the spend gate re-checks the cross-pod counter at pipeline entry, so it overshoots
+# only by the samples already in flight when the cap is crossed.
 _JOBS_CACHE_TTL_SECONDS: Final = 10
 
 # Concurrent shadow+judge pipelines per pod: a traffic spike turns into skipped samples
@@ -340,13 +341,24 @@ def _failure_detail(e: BaseException) -> str:
     return f"{type(e).__name__}{location}: {e}"
 
 
-def _judge_call_cost(response: object) -> float:
-    """Price a judge call, treating an unmapped judge model as free rather than fatal."""
+def _call_cost(response: object) -> float:
+    """Price one eval-arm call with the figure the spend pipeline bills: the router client
+    stamps _hidden_params.response_cost from the deployment's own pricing, which the public
+    price map lookup below cannot see (it reads 0 for deployment-priced models)."""
+    getter: Final = getattr(getattr(response, "_hidden_params", None), "get", None)
+    stamped: Final = getter("response_cost") if callable(getter) else None
+    if isinstance(stamped, (int, float)):
+        return float(stamped)
+    return _price_map_cost(response)
+
+
+def _price_map_cost(response: object) -> float:
+    """Public price map fallback, treating an unmapped model as free rather than fatal."""
     import litellm
 
     try:
         return litellm.completion_cost(completion_response=response) or 0.0
-    except Exception:  # noqa: BLE001  # unmapped judge model: the verdict still counts, cost stays 0
+    except Exception:  # noqa: BLE001  # unmapped model: the attempt still counts, cost stays 0
         return 0.0
 
 
@@ -374,6 +386,32 @@ def _judge_user_prompt(conversation: str, response_a: str, response_b: str) -> s
     )
 
 
+def _job_spend_counter_key(job_id: str) -> str:
+    return f"spend:shadow_eval:{job_id}"
+
+
+async def _job_spend_from_counter(counter_key: str, fallback_spend: float, max_budget: float) -> float:
+    """The leg's spend through the cross-pod counter the key budget gates read. The owner
+    degrades internally to the fill-time DB floor and raises only under fail-closed
+    enforcement, which the caller honors by skipping the sample."""
+    from litellm.proxy.proxy_server import get_current_spend
+
+    return await get_current_spend(counter_key=counter_key, fallback_spend=fallback_spend, max_budget=max_budget)
+
+
+async def _add_job_spend_to_counter(counter_key: str, cost: float) -> None:
+    """Advance the counter the moment a cost is known, so even a lost row closes the gate.
+    Known failure mode: a Redis outage freezes the counter (the owner invalidates it), the
+    gate degrades to the fill floor, and overshoot grows to in-flight plus one TTL of
+    samples, the same degradation the key budget counters accept."""
+    try:
+        from litellm.proxy.proxy_server import increment_spend_counter
+
+        await increment_spend_counter(counter_key=counter_key, increment=cost)
+    except Exception as e:  # noqa: BLE001  # attempt recording must proceed; the row stays truth and the fill floor gates
+        verbose_logger.warning("shadow_eval: spend counter increment failed for %s: %s", counter_key, e)
+
+
 async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
     """Whether the shadowed key or its team is over budget, decided by the same owners
     the request path uses, so counter keys and thresholds can never drift from auth's.
@@ -438,8 +476,8 @@ def _request_was_routed_by(request_metadata: Mapping[str, object], router_name:
 
 @dataclass(frozen=True, slots=True)
 class _CallFailure:
-    """A shadow or judge call that produced no usable response. cost carries any judge
-    spend the failed attempt still billed, so job-level judge_spend never undercounts."""
+    """A shadow or judge call that produced no usable response. cost carries any spend
+    the failed call still billed, so job-level spend figures never undercount."""
 
     error: str
     cost: float = 0.0
@@ -452,6 +490,7 @@ class _ShadowResponse:
     text: str
     model: str
     tier: str | None
+    cost: float
 
 
 @dataclass(frozen=True, slots=True)
@@ -478,8 +517,10 @@ class ActiveShadowEvalJob(BaseModel):
     shadow_percentage: float
     judge_model: str
     max_turns: int
+    max_budget: float | None = None
     ends_at: datetime
     attempts: int = 0
+    spend: float = 0.0
 
     @field_validator("ends_at")
     @classmethod
@@ -500,7 +541,7 @@ class ActiveShadowEvalJob(BaseModel):
         return self.baseline_model or self.router_name
 
 
-def _as_active_job(record: object, attempts: int) -> ActiveShadowEvalJob | None:
+def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None:
     """The sampling path's view of one job row, or None for a row it cannot sample: an
     unknown direction, or a reverse job with no baseline model to duplicate against.
     Failing closed here is what keeps the dispatch path total."""
@@ -509,7 +550,7 @@ def _as_active_job(record: object, attempts: int) -> ActiveShadowEvalJob | None:
     except ValidationError as e:
         verbose_logger.debug("shadow_eval: skipping unsamplable job row: %s", e)
         return None
-    return job.model_copy(update={"attempts": attempts})
+    return job.model_copy(update={"attempts": attempts, "spend": spend})  # mutable-ok: pydantic update payload
 
 
 _jobs_cache: Final = InMemoryCache(max_size_in_memory=4, default_ttl=_JOBS_CACHE_TTL_SECONDS)
@@ -524,12 +565,17 @@ class ShadowEvalLogger(CustomLogger):
         router_provider: Callable[[], "Router | None"] | None = None,
         prisma_provider: Callable[[], "PrismaClient | None"] | None = None,
         jobs_cache: InMemoryCache | None = None,
+        job_spend_reader: Callable[[str, float, float], Awaitable[float]] | None = None,
+        job_spend_writer: Callable[[str, float], Awaitable[None]] | None = None,
     ) -> None:
         """Providers are callables so the proxy's lazily-initialized globals are resolved
-        at call time, not at logger construction."""
+        at call time, not at logger construction. The spend reader and writer wrap the
+        proxy's cross-pod spend counter; tests inject a plain in-memory pair."""
         self._router_provider = router_provider or default_router_provider
         self._prisma_provider = prisma_provider or _default_prisma_provider
         self._jobs_cache = jobs_cache or _jobs_cache
+        self._read_job_spend = job_spend_reader or _job_spend_from_counter
+        self._write_job_spend = job_spend_writer or _add_job_spend_to_counter
         self._inflight_shadow_tasks: int = 0
         # Starts per job since the last cache fill, never decremented within a
         # generation; the refill absorbs written rows and resets.
@@ -556,18 +602,26 @@ class ShadowEvalLogger(CustomLogger):
                 await prisma.db.litellm_shadowevalattempt.group_by(
                     by=["job_id"],
                     count=True,
+                    sum={"judge_cost": True, "shadow_cost": True},  # mutable-ok: Prisma aggregate spec
                     where={"job_id": {"in": [str(record.id) for record in records]}},  # mutable-ok: Prisma filter
                 )
                 if records
                 else ()
             )
-            attempt_counts: Final = {str(row["job_id"]): int(row["_count"]["_all"]) for row in grouped or []}
+            attempt_stats: Final = {  # mutable-ok: frozen snapshot of the grouped read
+                str(row["job_id"]): (
+                    int(row["_count"]["_all"]),
+                    float((row["_sum"] or {}).get("judge_cost") or 0.0)
+                    + float((row["_sum"] or {}).get("shadow_cost") or 0.0),
+                )
+                for row in grouped or []
+            }
             by_key: Final = tuple(
                 sorted(
                     (
                         (str(record.api_key_id), job)
                         for record in records or []
-                        if (job := _as_active_job(record, attempt_counts.get(str(record.id), 0))) is not None
+                        if (job := _as_active_job(record, *attempt_stats.get(str(record.id), (0, 0.0)))) is not None
                     ),
                     key=itemgetter(0),
                 )
@@ -624,6 +678,7 @@ class ShadowEvalLogger(CustomLogger):
                 for job in (await self._active_jobs()).get(str(api_key_hash), ())
                 if datetime.now(timezone.utc) < job.ends_at
                 and job.attempts + self._job_starts.get(job.id, 0) < job.max_turns
+                and (job.max_budget is None or job.spend < job.max_budget)
                 and _sample_hits(request_id, job.id, job.shadow_percentage)
                 and _request_was_routed_by(request_metadata, job.router_name) == (job.direction == "reverse")
             )
@@ -684,12 +739,28 @@ class ShadowEvalLogger(CustomLogger):
                 return
             if await _key_or_team_is_over_budget(parent_metadata):
                 return
-
+            if job.max_budget is not None:
+                try:
+                    spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget)
+                except Exception as e:  # noqa: BLE001  # unverifiable budget: skip the sample rather than spend on it
+                    verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e)
+                    return
+                if spend >= job.max_budget:
+                    return
             shadow: Final = await self._call_router_shadow(job.shadow_target, messages, shadow_params, parent_metadata)
-            if isinstance(shadow, _CallFailure):
-                await self._record_attempt(prisma, job, request_id, control_tier, outcome="error", error=shadow.error)
-                return
-
+        except Exception as e:  # noqa: BLE001  # detached task: nothing billed yet, record and never raise
+            verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
+            await self._record_attempt(
+                prisma, job, request_id, control_tier, outcome="error", error=f"pipeline error: {e}"
+            )
+            return
+        if isinstance(shadow, _CallFailure):
+            await self._record_attempt(
+                prisma, job, request_id, control_tier, outcome="error", error=shadow.error, shadow_cost=shadow.cost
+            )
+            return
+        # From here the shadow call has billed, so every exit records its cost.
+        try:
             verdict: Final = await self._call_judge(
                 judge_model=job.judge_model,
                 messages=messages,
@@ -707,6 +778,7 @@ class ShadowEvalLogger(CustomLogger):
                     error=verdict.error,
                     shadow=shadow,
                     judge_cost=verdict.cost,
+                    shadow_cost=shadow.cost,
                 )
                 return
             await self._record_attempt(
@@ -719,15 +791,23 @@ class ShadowEvalLogger(CustomLogger):
                 real_model=real_model,
                 confidence=verdict.confidence,
                 judge_cost=verdict.cost,
+                shadow_cost=shadow.cost,
             )
-        except Exception as e:  # noqa: BLE001  # detached task: record what happened, never raise
+        except Exception as e:  # noqa: BLE001  # detached task: the shadow call billed, record its cost, never raise
             verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
             await self._record_attempt(
-                prisma, job, request_id, control_tier, outcome="error", error=f"pipeline error: {e}"
+                prisma,
+                job,
+                request_id,
+                control_tier,
+                outcome="error",
+                error=f"pipeline error: {e}",
+                shadow=shadow,
+                shadow_cost=shadow.cost,
             )
 
-    @staticmethod
     async def _record_attempt(
+        self,
         prisma: "PrismaClient | None",
         job: ActiveShadowEvalJob,
         request_id: str,
@@ -738,8 +818,11 @@ class ShadowEvalLogger(CustomLogger):
         real_model: str = "",
         confidence: float | None = None,
         judge_cost: float = 0.0,
+        shadow_cost: float = 0.0,
         error: str | None = None,
     ) -> None:
+        if judge_cost + shadow_cost > 0:
+            await self._write_job_spend(_job_spend_counter_key(job.id), judge_cost + shadow_cost)
         if prisma is None:
             return
         try:
@@ -753,6 +836,7 @@ class ShadowEvalLogger(CustomLogger):
                     "shadow_model": shadow.model if shadow else None,
                     "confidence": confidence,
                     "judge_cost": judge_cost,
+                    "shadow_cost": shadow_cost,
                     "error": error[:_MAX_ERROR_CHARS] if error else None,
                 }
             )
@@ -792,11 +876,12 @@ class ShadowEvalLogger(CustomLogger):
             return _CallFailure(f"shadow router call failed: {_failure_detail(e)}")
         text: Final = _chat_final_text(response)
         if not text:
-            return _CallFailure("shadow router returned an empty response")
+            return _CallFailure("shadow router returned an empty response", cost=_call_cost(response))
         return _ShadowResponse(
             text=text,
             model=str(getattr(response, "model", None) or _routing_decision(shadow_metadata).get("routed_model") or ""),
             tier=_routed_tier(shadow_metadata),
+            cost=_call_cost(response),
         )
 
     async def _call_judge(
@@ -843,11 +928,11 @@ class ShadowEvalLogger(CustomLogger):
             verdict: Final = PairwiseVerdict.model_validate(parse_json_verdict(raw))
         except Exception as e:  # noqa: BLE001  # malformed verdicts become error rows
             verbose_logger.debug("shadow_eval: unparseable judge verdict: %s", e)
-            return _CallFailure(f"unparseable judge verdict: {e}", cost=_judge_call_cost(response))
+            return _CallFailure(f"unparseable judge verdict: {e}", cost=_call_cost(response))
         return _JudgeVerdict(
             preference=_unmask_preference(verdict.preference, real_is_a),
             confidence=max(0.0, min(1.0, verdict.confidence)),
-            cost=_judge_call_cost(response),
+            cost=_call_cost(response),
         )
 
 

litellm/litellm_core_utils/cli_token_utils.py

@@ -4,10 +4,10 @@ CLI Token Utilities
 SDK-level utilities for reading the credential minted by `lite login`.
 
 Non-secret metadata lives in ~/.litellm/token.json. The secret material (the
-bearer key, plus a JWT when one is issued) lives in the OS keychain when the
-machine has one, and in that same 0600 file otherwise. This module hides the
-split from callers, and migrates a legacy plaintext file into the keychain the
-first time it reads one.
+bearer key, the refresh token that renews it, and a JWT when one is issued)
+lives in the OS keychain when the machine has one, and in that same 0600 file
+otherwise. This module hides the split from callers, and migrates a plaintext
+file into the keychain the first time it reads one.
 
 This module has no dependencies on proxy code and can be safely imported at the SDK level.
 """
@@ -111,13 +111,20 @@ class CliTokenSecret(BaseModel):
     secret minted for one server is never handed to another, even if the
     metadata file is edited underneath us. `timestamp` is the sign-in this
     secret came from, which is what decides it against a secret still on disk.
+
+    Every field a thief could sign in with belongs here, which is why the
+    refresh token is one of them: it buys a fresh key from the proxy on demand,
+    so leaving it on disk would leave the login readable there. `key` is
+    optional because the file can hold a refresh token without one, and moving
+    that into the keychain must not invent a key to go with it.
     """
 
     model_config = ConfigDict(frozen=True)
 
     base_url: str
-    key: str
+    key: str | None = None
     jwt_token: str = ""
+    refresh_token: str | None = None
     timestamp: float = 0.0
 
 
@@ -155,7 +162,7 @@ def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRIN
     staged: Final = _stage_token_file(_without_secret(stamped))
     if isinstance(staged, CredentialNotSaved):
         return staged
-    outcome: Final = SecretStored() if stamped.key is None else vault.write(_encode_secret(stamped, stamped.key))
+    outcome: Final = vault.write(_encode_secret(stamped)) if _holds_a_secret(stamped) else SecretStored()
     if isinstance(outcome, SecretStored):
         return outcome if _commit_token_file(staged) else CredentialNotRecorded()
     discard_staged_json(staged)
@@ -390,8 +397,9 @@ def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) -
     secret is usually left on disk by a keychain that would not take it, which makes the file the
     fresher of the two. It is the older one when a login the keychain did take could not replace
     the file afterwards, and serving that one would put a superseded credential back in use. Equal
-    stamps are one login sitting in both stores, left by a migration whose scrub was refused, so
-    that branch retries the migration rather than trading one credential for another.
+    stamps are one login sitting in both stores, left by a migration whose scrub was refused or by
+    an upgrade that took the key into the keychain and left the refresh token behind, so that branch
+    rejoins the halves and retries the migration rather than trading one credential for another.
 
     A scrub the file refuses leaves that superseded secret where it lies, which is the state the
     login already named when it could not replace the file, and which `lite logout` reports rather
@@ -400,21 +408,46 @@ def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) -
     superseded one back out.
     """
     secret: Final = _decode_secret(blob, record.base_url)
-    if secret is None or (record.key is not None and secret.timestamp <= record.timestamp):
-        return _migrate_file_secret(record, vault)
+    if secret is None or (_holds_a_secret(record) and secret.timestamp <= record.timestamp):
+        return _migrate_file_secret(_rejoined(record, secret), vault, replacing=secret)
     _scrub_file_secret(record)
     return record.model_copy(
         update=MappingProxyType(
             {
                 "key": secret.key,
                 "jwt_token": secret.jwt_token,
+                "refresh_token": secret.refresh_token,
                 "timestamp": max(secret.timestamp, record.timestamp),
             }
         )
     )
 
 
-def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None:
+def _rejoined(record: CliTokenRecord, secret: CliTokenSecret | None) -> CliTokenRecord:
+    """Put one sign-in's secret material back together when each store holds part of it.
+
+    Upgrading from the release that kept only the key in the keychain leaves the refresh token
+    behind in the file, so a single login sits across both stores. Filling in whatever the file is
+    missing before the migration writes its entry is what stops that write from replacing a live key
+    with nothing. Only a matching stamp is one login. Two stamps are two logins, and pairing one's
+    key with the other's refresh token would build a credential neither store ever held.
+    """
+    if secret is None or secret.timestamp != record.timestamp:
+        return record
+    return record.model_copy(
+        update=MappingProxyType(
+            {
+                "key": record.key if record.key is not None else secret.key,
+                "jwt_token": record.jwt_token or secret.jwt_token,
+                "refresh_token": record.refresh_token if record.refresh_token is not None else secret.refresh_token,
+            }
+        )
+    )
+
+
+def _migrate_file_secret(
+    record: CliTokenRecord, vault: SecretVault, *, replacing: CliTokenSecret | None = None
+) -> CliTokenRecord | None:
     """Move a file-held secret into the vault, but only once the file's copy can be taken away.
 
     The scrubbed file is staged first so a directory that will not accept it stops the migration
@@ -426,23 +459,28 @@ def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliToken
     asked to take the new entry back, so the migration finishes on a directory that would only ever
     have refused it. Rolling back is the last resort, and a rollback the keychain also refuses
     leaves the secret in both stores until the next read, which retries this same migration.
+
+    Only an entry this migration put there is taken back. `replacing` names one that was already in
+    the keychain, whose material the new entry carries forward, so erasing it would take away the
+    half the file never had, and a machine that refuses the scrub is exactly the one with nowhere
+    else to keep it. The next read finds the same two halves and tries the move again.
     """
-    if record.key is None:
+    if not _holds_a_secret(record):
         return None
     staged: Final = _stage_scrubbed_file(record)
     if staged is None:
         return record
-    if not isinstance(vault.write(_encode_secret(record, record.key)), SecretStored):
+    if not isinstance(vault.write(_encode_secret(record)), SecretStored):
         discard_staged_json(staged)
         return record
-    if not _commit_token_file(staged) and not _overwrite_file_secret(record):
+    if not _commit_token_file(staged) and not _overwrite_file_secret(record) and replacing is None:
         vault.erase()
     return record
 
 
 def _scrub_file_secret(record: CliTokenRecord) -> bool:
     """Leave no secret material in the token file once the vault holds it"""
-    if record.key is None and not record.jwt_token:
+    if not _holds_a_secret(record):
         return True
     staged: Final = _stage_scrubbed_file(record)
     if staged is not None and _commit_token_file(staged):
@@ -487,13 +525,22 @@ def _commit_token_file(staged: str) -> bool:
     return True
 
 
+def _holds_a_secret(record: CliTokenRecord) -> bool:
+    """Whether the record carries anything that would sign someone in as this user"""
+    return record.key is not None or bool(record.jwt_token) or record.refresh_token is not None
+
+
 def _without_secret(record: CliTokenRecord) -> CliTokenRecord:
-    return record.model_copy(update=MappingProxyType({"key": None, "jwt_token": ""}))
+    return record.model_copy(update=MappingProxyType({"key": None, "jwt_token": "", "refresh_token": None}))
 
 
-def _encode_secret(record: CliTokenRecord, key: str) -> str:
+def _encode_secret(record: CliTokenRecord) -> str:
     return CliTokenSecret(
-        base_url=record.base_url, key=key, jwt_token=record.jwt_token, timestamp=record.timestamp
+        base_url=record.base_url,
+        key=record.key,
+        jwt_token=record.jwt_token,
+        refresh_token=record.refresh_token,
+        timestamp=record.timestamp,
     ).model_dump_json()
 
 

litellm/litellm_core_utils/env_utils.py

@@ -2,6 +2,7 @@
 Utility helpers for reading and parsing environment variables.
 """
 
+import logging
 import os
 from typing import Final
 
@@ -22,6 +23,26 @@ def get_env_int(env_var: str, default: int) -> int:
         return default
 
 
+def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int) -> int:
+    """Parse an environment variable as an integer constrained to ``[minimum, maximum]``.
+
+    Values outside the range fall back to the default and warn, so a misconfigured knob can
+    neither crash the caller nor silently change the meaning of what it computes.
+    """
+    value: Final = get_env_int(env_var, default)
+    if minimum <= value <= maximum:
+        return value
+    logging.getLogger("LiteLLM").warning(
+        "%s=%s is outside the supported range [%s, %s]. Falling back to %s.",
+        env_var,
+        value,
+        minimum,
+        maximum,
+        default,
+    )
+    return default
+
+
 def get_env_int_or_none(env_var: str) -> int | None:
     """Parse an environment variable as an integer, returning None when it is unset or unusable.
 

litellm/litellm_core_utils/get_litellm_params.py

@@ -21,6 +21,14 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
     }
 )
 
+# The per-deployment Rust opt-in.
+RUST_KWARG_KEY: Final = "rust"
+
+# Keys `completion()` forwards from its own kwargs into `get_litellm_params`,
+# which are otherwise invisible to it because that call site passes explicit
+# named arguments rather than `**kwargs`.
+FORWARDED_KWARGS_KEYS: Final = AWS_CREDENTIAL_KWARGS_KEYS | frozenset({RUST_KWARG_KEY})
+
 # Pre-define optional kwargs keys as frozenset for O(1) lookups
 # These are extracted from kwargs only if present, avoiding unnecessary .get() calls
 OPTIONAL_KWARGS_KEYS: Final = (
@@ -47,6 +55,10 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "itpm",
             "otpm",
             "use_xai_oauth",
+            # The per-deployment Rust opt-in. `all_litellm_params` keeps it out
+            # of the provider body; this keeps it *in* litellm_params, which is
+            # where the chat completions handlers read it from.
+            RUST_KWARG_KEY,
         }
     )
     | AWS_CREDENTIAL_KWARGS_KEYS

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -349,6 +349,9 @@ def get_llm_provider(
                     elif endpoint == "https://api.meta.ai/v1":
                         custom_llm_provider = "meta"
                         dynamic_api_key = get_secret_str("META_API_KEY")
+                    elif (json_provider := JSONProviderRegistry.get_by_base_url(endpoint)) is not None:
+                        custom_llm_provider = json_provider.slug
+                        dynamic_api_key = api_key if api_key is not None else get_secret_str(json_provider.api_key_env)
 
                     if api_base is not None and not isinstance(api_base, str):
                         raise Exception(f"api base needs to be a string. api_base={api_base}")

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1371,6 +1371,7 @@ class CostCalculatorUtils:
             return fal_ai_image_cost_calculator(
                 model=model,
                 image_response=completion_response,
+                optional_params=optional_params,
             )
         elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
             from litellm.llms.runwayml.cost_calculator import (

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1200,13 +1200,14 @@ def _encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: st
     return tool_call_id
 
 
-def _get_thought_signature_from_tool(tool: dict, model: str | None = None) -> str | None:
+def _get_thought_signature_from_tool(tool: dict) -> str | None:
     """Extract thought signature from tool call's provider_specific_fields.
 
     If not provided try to extract thought signature from tool call id
 
     Checks both tool.provider_specific_fields and tool.function.provider_specific_fields.
-    If no signature is found and model is gemini-3, returns a dummy signature.
+    Returns None when the tool call carries no signature; callers decide whether a
+    placeholder signature is needed.
     """
     # First check tool's provider_specific_fields
     provider_fields: Final = tool.get("provider_specific_fields") or {}
@@ -1236,13 +1237,6 @@ def _get_thought_signature_from_tool(tool: dict, model: str | None = None) -> st
         if len(parts) == 2:
             _, signature = parts
             return signature
-    # If no signature found and model is gemini-3, return dummy signature
-    from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
-        VertexGeminiConfig,
-    )
-
-    if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
-        return _get_dummy_thought_signature()
     return None
 
 
@@ -1251,10 +1245,14 @@ def _get_dummy_thought_signature() -> str:
 
     This is used when transferring conversation history from older models
     (like gemini-2.5-flash) to gemini-3, which requires thought_signature
-    for strict validation.
+    for strict validation. Google documents it as a last resort that "will
+    negatively impact model performance", so callers must only fall back to it
+    when no real signature is available.
+
+    See:
+    https://ai.google.dev/gemini-api/docs/thought-signatures#faqs
+    https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures
     """
-    # Return a base64-encoded dummy signature string
-    # Below dummy signature is recommended by google - https://ai.google.dev/gemini-api/docs/thought-signatures#faqs
     dummy_data: Final = b"skip_thought_signature_validator"
     return base64.b64encode(dummy_data).decode("utf-8")
 
@@ -1312,8 +1310,10 @@ def convert_to_gemini_tool_call_invoke(
             VertexGeminiConfig,
         )
 
+        needs_dummy_signature: Final = model is not None and VertexGeminiConfig._is_gemini_3_or_newer(model)
+
         if tool_calls is not None:
-            for idx, tool in enumerate(tool_calls):
+            for tool in tool_calls:
                 if "function" in tool:
                     gemini_function_call: VertexFunctionCall | None = _gemini_tool_call_invoke_helper(
                         function_call_params=tool["function"],
@@ -1321,7 +1321,13 @@ def convert_to_gemini_tool_call_invoke(
                     )
                     if gemini_function_call is not None:
                         part_dict: VertexPartType = {"function_call": gemini_function_call}
-                        thought_signature = _get_thought_signature_from_tool(dict(tool), model=model)
+                        thought_signature = _get_thought_signature_from_tool(dict(tool))
+                        # Gemini signs only the first functionCall part of a parallel batch, so scope the
+                        # placeholder fallback to that part instead of fabricating one per sibling call:
+                        # https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures#parallel_function_calling_example
+                        is_first_function_call = len(_parts_list) == 0
+                        if not thought_signature and is_first_function_call and needs_dummy_signature:
+                            thought_signature = _get_dummy_thought_signature()
                         if thought_signature:
                             part_dict["thoughtSignature"] = thought_signature
 
@@ -1344,7 +1350,7 @@ def convert_to_gemini_tool_call_invoke(
                     thought_signature = provider_fields.get("thought_signature")
 
                 # If no signature found and model is gemini-3, use dummy signature
-                if not thought_signature and model and VertexGeminiConfig._is_gemini_3_or_newer(model):
+                if not thought_signature and needs_dummy_signature:
                     thought_signature = _get_dummy_thought_signature()
 
                 if thought_signature:

litellm/litellm_core_utils/ptu_pricing.py

@@ -79,6 +79,45 @@ def _as_utc(value: object) -> datetime | None:
         return None
 
 
+def _named(reason: str, model_name: str | None) -> str:
+    """The reason on its own for a caller that already has the deployment in hand, else named."""
+    return reason if model_name is None else f"PTU configuration on model '{model_name}' is invalid: {reason}"
+
+
+def ptu_config_error(model_info: Mapping[str, object], *, model_name: str | None = None) -> str | None:
+    """Why this PTU configuration cannot be honoured, else None.
+
+    Both the model endpoints and config.yaml registration ask this, so a deployment that
+    one refuses is refused by the other for the same stated reason.
+
+    Window ordering is checked before the count/rate gate. A patch that touches only one end
+    of the window carries no count or rate, so leaving the order to that gate would let an
+    inverted window reach the row; the next load then fails to parse it and drops the
+    deployment out of the router, where no further patch can repair it.
+    """
+    effective_from: Final = _as_utc(model_info.get("ptu_effective_from"))
+    effective_to: Final = _as_utc(model_info.get("ptu_effective_to"))
+    if effective_from is not None and effective_to is not None and effective_to <= effective_from:
+        return _named("ptu_effective_to must be after ptu_effective_from", model_name)
+
+    has_count: Final = model_info.get("ptu_count") is not None
+    has_rate: Final = model_info.get("cost_per_ptu_per_hour") is not None
+    if not has_count and not has_rate:
+        return None
+    if has_count != has_rate:
+        return _named("ptu_count and cost_per_ptu_per_hour must be set together", model_name)
+    if effective_from is None:
+        return _named(
+            "ptu_effective_from is required when PTU fields are set. Flat cost accrues from that "
+            "instant, so without it the start would have to be inferred and a deployment configured "
+            "today could be billed for days it did not exist",
+            model_name,
+        )
+    if not model_info.get("team_id"):
+        return _named("team_id is required when PTU fields are set (one model maps to one team)", model_name)
+    return None
+
+
 def ptu_terms(model_info: Mapping[str, object]) -> PTUTerms | None:
     """The reservation this deployment accrues flat cost for, else None.
 

litellm/litellm_core_utils/streaming_handler.py

@@ -2315,10 +2315,18 @@ class CustomStreamWrapper:
         if self.logging_obj is None or not self.chunks:
             return
         try:
-            partial_response: Final = litellm.stream_chunk_builder(chunks=self.chunks)
+            partial_response: Final = litellm.stream_chunk_builder(
+                chunks=self.chunks,
+                messages=self.messages if isinstance(self.messages, list) else None,
+            )
+            if partial_response is None:
+                return
             usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
             if usage is None:
                 return
+            if self.model:
+                partial_response.model = self.model
+            backfill_missing_cache_usage_fields(usage)
             self.logging_obj.model_call_details["combined_usage_object"] = usage
             self.logging_obj.model_call_details["response_cost"] = (
                 self.logging_obj._response_cost_calculator(result=partial_response) or 0.0
@@ -2439,6 +2447,35 @@ class CustomStreamWrapper:
         return chunk
 
 
+def _cache_token_count(details: PromptTokensDetailsWrapper | None, keys: tuple[str, ...]) -> int:
+    for key in keys:
+        value = getattr(details, key, None)
+        if isinstance(value, int) and not isinstance(value, bool) and value:
+            return value
+    return 0
+
+
+def backfill_missing_cache_usage_fields(usage: Usage) -> None:
+    """Give partial-stream usage the same cache fields a complete stream reports.
+
+    Carries OpenAI-style ``prompt_tokens_details`` counts up to the Anthropic-style
+    top-level keys, defaulting to zero. It must carry the real count rather than a
+    flat zero: downstream readers treat these keys as authoritative once present and
+    skip their own normalization, so a zero here would overwrite a real cache read.
+    """
+    details: Final = usage.prompt_tokens_details
+    if getattr(usage, "cache_read_input_tokens", None) is None:
+        usage.cache_read_input_tokens = _cache_token_count(  # rebind-ok: in-place backfill is the contract
+            details, ("cached_tokens",)
+        )
+    if getattr(usage, "cache_creation_input_tokens", None) is None:
+        usage.cache_creation_input_tokens = _cache_token_count(  # rebind-ok: in-place backfill is the contract
+            details, ("cache_write_tokens", "cache_creation_tokens")
+        )
+    if usage.prompt_tokens_details is None:
+        usage.prompt_tokens_details = PromptTokensDetailsWrapper(cached_tokens=0)  # rebind-ok: backfill in place
+
+
 _TokenDetails = TypeVar("_TokenDetails", PromptTokensDetailsWrapper, CompletionTokensDetailsWrapper)
 
 

litellm/litellm_core_utils/thread_pool_executor.py

@@ -1,6 +1,82 @@
-from concurrent.futures import ThreadPoolExecutor
-from typing import Final
+import logging
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from typing import Final, ParamSpec, TypeVar
 
-MAX_THREADS: Final = 100
-# Create a ThreadPoolExecutor
-executor: Final = ThreadPoolExecutor(max_workers=MAX_THREADS)
+from litellm._logging import verbose_logger
+from litellm.constants import (
+    LOGGING_EXECUTOR_DROPPED_TASK_LOG_INTERVAL_SECONDS,
+    LOGGING_EXECUTOR_MAX_PENDING_TASKS,
+    LOGGING_EXECUTOR_MAX_THREADS,
+)
+
+MAX_THREADS: Final = LOGGING_EXECUTOR_MAX_THREADS
+
+_P = ParamSpec("_P")
+_T = TypeVar("_T")
+
+
+class BoundedLoggingThreadPoolExecutor(ThreadPoolExecutor):
+    """ThreadPoolExecutor with a cap on queued-plus-running tasks.
+
+    The default ThreadPoolExecutor work queue is unbounded, and every queued
+    logging task pins its request/response payload in memory, so a sustained
+    burst of sync callbacks slower than request arrival grows memory without
+    bound. Logging is best-effort: once the cap is reached, new submissions
+    are dropped with a rate-limited warning instead of queueing forever.
+    """
+
+    def __init__(
+        self,
+        max_workers: int,
+        max_pending_tasks: int,
+        drop_log_interval_seconds: float = LOGGING_EXECUTOR_DROPPED_TASK_LOG_INTERVAL_SECONDS,
+        logger: logging.Logger = verbose_logger,
+    ) -> None:
+        super().__init__(max_workers=max_workers, thread_name_prefix="litellm-logging")
+        self._max_pending_tasks: Final = max_pending_tasks
+        self._drop_log_interval_seconds: Final = drop_log_interval_seconds
+        self._logger: Final = logger
+        self._pending_slots: Final = threading.Semaphore(max_pending_tasks)
+        self._drop_lock: Final = threading.Lock()
+        self._dropped_since_last_log = 0
+        self._last_drop_log_time = 0.0
+
+    def submit(self, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs) -> Future[_T]:
+        if not self._pending_slots.acquire(blocking=False):
+            self._record_drop()
+            dropped_future: Final[Future[_T]] = Future()
+            dropped_future.cancel()
+            return dropped_future
+        try:
+            future: Final = super().submit(fn, *args, **kwargs)
+        except BaseException:
+            self._pending_slots.release()
+            raise
+        future.add_done_callback(lambda _: self._pending_slots.release())
+        return future
+
+    def _record_drop(self) -> None:
+        with self._drop_lock:
+            self._dropped_since_last_log += 1
+            now: Final = time.monotonic()
+            if now - self._last_drop_log_time < self._drop_log_interval_seconds:
+                return
+            dropped_count: Final = self._dropped_since_last_log
+            self._dropped_since_last_log = 0
+            self._last_drop_log_time = now
+
+        self._logger.warning(
+            "litellm logging executor backlog is full (max_pending_tasks=%s); dropped %s logging task(s) "
+            "since the last warning. Set LOGGING_EXECUTOR_MAX_PENDING_TASKS to raise the cap.",
+            self._max_pending_tasks,
+            dropped_count,
+        )
+
+
+executor: Final = BoundedLoggingThreadPoolExecutor(
+    max_workers=MAX_THREADS,
+    max_pending_tasks=LOGGING_EXECUTOR_MAX_PENDING_TASKS,
+)

litellm/litellm_core_utils/token_counter.py

@@ -19,6 +19,7 @@ from litellm.constants import (
     MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES,
     MAX_TILE_HEIGHT,
     MAX_TILE_WIDTH,
+    TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS,
 )
 from litellm.litellm_core_utils.default_encoding import encoding as default_encoding
 from litellm.litellm_core_utils.url_utils import safe_get
@@ -305,6 +306,16 @@ Type for a function that counts tokens in a string.
 """
 
 
+def _get_tiktoken_count_function(
+    encode_length: Callable[[str], int],
+    chunk_size: int = TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS,
+) -> TokenCounterFunction:
+    def count_tokens(text: str) -> int:
+        return sum(encode_length(text[start : start + chunk_size]) for start in range(0, len(text), chunk_size))
+
+    return count_tokens
+
+
 class _MessageCountParams:
     """
     A class to hold the parameters for counting tokens in messages.
@@ -531,6 +542,7 @@ def _get_count_function(
                 enc: Final = tokenizer_json["tokenizer"].encode(text)
                 return len(enc.ids)
 
+            return count_tokens
         elif tokenizer_json["type"] == "openai_tokenizer":
             model_to_use: Final = _fix_model_name(model)
             try:
@@ -542,17 +554,18 @@ def _get_count_function(
                 print_verbose("Warning: model not found. Using cl100k_base encoding.")
                 encoding = tiktoken.get_encoding("cl100k_base")
 
-            def count_tokens(text: str) -> int:
+            def encode_length(text: str) -> int:
                 return len(encoding.encode(text, disallowed_special=()))
 
+            return _get_tiktoken_count_function(encode_length)
         else:
             raise ValueError("Unsupported tokenizer type")
     else:
 
-        def count_tokens(text: str) -> int:
+        def encode_length(text: str) -> int:
             return len(default_encoding.encode(text, disallowed_special=()))
 
-    return count_tokens
+        return _get_tiktoken_count_function(encode_length)
 
 
 def _fix_model_name(model: str) -> str:

litellm/llms/anthropic/chat/handler.py

@@ -24,6 +24,8 @@ from litellm.llms.custom_httpx.http_handler import (
     _get_httpx_client,
     get_async_httpx_client,
 )
+from litellm.rust_bridge import chat_completions as rust_chat_completions_bridge
+from litellm.rust_bridge.chat_completions import rust_chat_completions_accepts
 from litellm.types.llms.anthropic import (
     ContentBlockDelta,
     ContentBlockStart,
@@ -361,30 +363,135 @@ class AnthropicChatCompletion(BaseLLM):
         if config is None:
             raise ValueError(f"Provider config not found for model: {model} and provider: {custom_llm_provider}")
 
-        data = config.transform_request(
+        def build_request() -> tuple[dict, dict]:  # mutable-ok: rewritten in place downstream
+            """Translate the request the Python way, returning `(headers, data)`.
+
+            The pair stays mutable because the streaming path rewrites it in
+            place (`data["stream"] = True`) before sending.
+
+            Shared by the normal path and by the Rust path's fallback, which
+            builds it only when the Rust call did not serve the request.
+            """
+            request_data: Final = config.transform_request(
+                model=model,
+                messages=messages,
+                optional_params={**optional_params, "is_vertex_request": is_vertex_request},
+                litellm_params=litellm_params,
+                headers=headers,
+            )
+            return update_request_with_filtered_beta(
+                headers=headers,
+                request_data=request_data,
+                provider=custom_llm_provider,
+            )
+
+        # The Rust core owns the whole call for the subset it accepts, so ask
+        # before transforming: whichever path runs emits pre_call exactly once.
+        # `get_config` merges the class-level defaults (Anthropic's required
+        # `max_tokens` among them) that `transform_request` would have applied.
+        rust_optional_params: Final = {  # mutable-ok: json.dumps in the bridge rejects a mappingproxy
+            **AnthropicConfig.get_config(model=model),
+            **optional_params,
+        }
+        serves_via_rust: Final = rust_chat_completions_accepts(
             model=model,
             messages=messages,
-            optional_params={**optional_params, "is_vertex_request": is_vertex_request},
+            optional_params=rust_optional_params,
+            custom_llm_provider=custom_llm_provider,
             litellm_params=litellm_params,
-            headers=headers,
+            stream=stream,
         )
-
-        headers, data = update_request_with_filtered_beta(
-            headers=headers,
-            request_data=data,
-            provider=custom_llm_provider,
-        )
-
-        ## LOGGING
-        logging_obj.pre_call(
-            input=messages,
-            api_key=api_key,
-            additional_args={
-                "complete_input_dict": data,
+        if serves_via_rust:
+            rust_logging_args: Final = {  # mutable-ok: logging callbacks read additional_args as a plain dict
+                "complete_input_dict": {  # mutable-ok: same, and it is serialized alongside its parent
+                    "model": model,
+                    "messages": messages,
+                    **rust_optional_params,
+                },
                 "api_base": api_base,
                 "headers": headers,
-            },
-        )
+            }
+            logging_obj.pre_call(input=messages, api_key=api_key, additional_args=rust_logging_args)
+            log_rust_post_call: Final = rust_chat_completions_bridge.response_logger(
+                logging_obj=logging_obj,
+                messages=messages,
+                api_key=api_key,
+                additional_args=rust_logging_args,
+            )
+            if acompletion is True:
+
+                async def python_fallback() -> "ModelResponse | CustomStreamWrapper":
+                    # pre_call already fired for this request above. The Rust
+                    # path only declines before the provider is called, so this
+                    # is the same attempt continuing, not a second one.
+                    fallback_headers, fallback_data = build_request()
+                    return await self.acompletion_function(
+                        model=model,
+                        messages=messages,
+                        data=fallback_data,
+                        api_base=api_base,
+                        custom_prompt_dict=custom_prompt_dict,
+                        model_response=model_response,
+                        print_verbose=print_verbose,
+                        encoding=encoding,
+                        api_key=api_key,
+                        provider_config=config,
+                        logging_obj=logging_obj,
+                        optional_params=optional_params,
+                        stream=stream,
+                        _is_function_call=_is_function_call,
+                        litellm_params=litellm_params,
+                        logger_fn=logger_fn,
+                        headers=fallback_headers,
+                        client=client,
+                        json_mode=json_mode,
+                        timeout=timeout,
+                    )
+
+                return rust_chat_completions_bridge.achat_completions_or_fallback(
+                    model=model,
+                    messages=messages,
+                    optional_params=rust_optional_params,
+                    model_response=model_response,
+                    api_key=api_key,
+                    api_base=api_base,
+                    custom_llm_provider=custom_llm_provider,
+                    extra_headers=headers,
+                    timeout=timeout,
+                    on_response=log_rust_post_call,
+                    python_fallback=python_fallback,
+                )
+            rust_response: Final = rust_chat_completions_bridge.chat_completions(
+                model=model,
+                messages=messages,
+                optional_params=rust_optional_params,
+                model_response=model_response,
+                api_key=api_key,
+                api_base=api_base,
+                custom_llm_provider=custom_llm_provider,
+                extra_headers=headers,
+                timeout=timeout,
+                on_response=log_rust_post_call,
+            )
+            if rust_response is not None:
+                return rust_response
+
+        headers, data = build_request()
+
+        ## LOGGING
+        # Reaching here with `serves_via_rust` set means the Rust attempt
+        # declined at call time, before the provider was called, and already
+        # logged this request. That is the same attempt continuing.
+        if not serves_via_rust:
+            logging_obj.pre_call(
+                input=messages,
+                api_key=api_key,
+                additional_args={
+                    "complete_input_dict": data,
+                    "api_base": api_base,
+                    "headers": headers,
+                },
+            )
         print_verbose(f"_is_function_call: {_is_function_call}")
         if acompletion is True:
             if (

litellm/llms/anthropic/chat/transformation.py

@@ -2216,7 +2216,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
     def calculate_usage(
         self,
-        usage_object: dict,
+        usage_object: Mapping[str, Any],
         reasoning_content: str | None,
         completion_response: dict | None = None,
         speed: str | None = None,

litellm/llms/bedrock/chat/converse_handler.py

@@ -14,6 +14,8 @@ from litellm.llms.custom_httpx.http_handler import (
     _get_httpx_client,
     get_async_httpx_client,
 )
+from litellm.rust_bridge import chat_completions as rust_chat_completions_bridge
+from litellm.rust_bridge.chat_completions import rust_chat_completions_accepts
 from litellm.types.utils import ModelResponse
 from litellm.utils import CustomStreamWrapper
 
@@ -169,6 +171,7 @@ class BedrockConverseLLM(BaseAWSLLM):
         headers: dict = {},
         client: AsyncHTTPHandler | None = None,
         api_key: str | None = None,
+        skip_pre_call_logging: bool = False,
     ) -> ModelResponse | CustomStreamWrapper:
         request_data: Final = await litellm.AmazonConverseConfig()._async_transform_request(
             model=model,
@@ -190,15 +193,19 @@ class BedrockConverseLLM(BaseAWSLLM):
         )
 
         ## LOGGING
-        logging_obj.pre_call(
-            input=messages,
-            api_key="",
-            additional_args={
-                "complete_input_dict": data,
-                "api_base": api_base,
-                "headers": prepped.headers,
-            },
-        )
+        # The Rust path already logged this request's pre_call before handing
+        # it here, and it only declines before the provider is called, so this
+        # is the same attempt continuing rather than a second one.
+        if not skip_pre_call_logging:
+            logging_obj.pre_call(
+                input=messages,
+                api_key="",
+                additional_args={
+                    "complete_input_dict": data,
+                    "api_base": api_base,
+                    "headers": prepped.headers,
+                },
+            )
 
         headers = dict(prepped.headers)
         if client is None or not isinstance(client, AsyncHTTPHandler):
@@ -354,6 +361,94 @@ class BedrockConverseLLM(BaseAWSLLM):
 
         # Filter beta headers in HTTP headers before making the request
         headers = update_headers_with_filtered_beta(headers=headers, provider="bedrock_converse")
+
+        # The Rust core owns the whole call for the subset it accepts. Ask
+        # before transforming so whichever path runs emits pre_call once, and
+        # hand down the credentials, region and endpoint this handler already
+        # resolved so both paths sign as the same principal.
+        rust_optional_params: Final = {  # mutable-ok: json.dumps in the bridge rejects a mappingproxy
+            **optional_params,
+            **{  # mutable-ok: merged into its mutable parent above
+                key: value
+                for key, value in (
+                    ("aws_access_key_id", credentials.access_key),
+                    ("aws_secret_access_key", credentials.secret_key),
+                    ("aws_session_token", credentials.token),
+                    ("aws_region_name", aws_region_name),
+                )
+                if value is not None
+            },
+        }
+        serves_via_rust: Final = rust_chat_completions_accepts(
+            model=model,
+            messages=messages,
+            optional_params=rust_optional_params,
+            custom_llm_provider="bedrock",
+            litellm_params=litellm_params,
+            stream=stream,
+        )
+        if serves_via_rust:
+            rust_logging_args: Final = {  # mutable-ok: logging callbacks read additional_args as a plain dict
+                "complete_input_dict": {  # mutable-ok: same, and it is serialized alongside its parent
+                    "messages": messages,
+                    **optional_params,
+                },
+                "api_base": proxy_endpoint_url,
+                "headers": headers,
+            }
+            logging_obj.pre_call(input=messages, api_key="", additional_args=rust_logging_args)
+            log_rust_post_call: Final = rust_chat_completions_bridge.response_logger(
+                logging_obj=logging_obj,
+                messages=messages,
+                api_key="",
+                additional_args=rust_logging_args,
+            )
+            if acompletion:
+                return rust_chat_completions_bridge.achat_completions_or_fallback(
+                    model=model,
+                    messages=messages,
+                    optional_params=rust_optional_params,
+                    model_response=model_response,
+                    api_key=api_key,
+                    api_base=proxy_endpoint_url,
+                    custom_llm_provider="bedrock",
+                    extra_headers=headers,
+                    timeout=timeout,
+                    on_response=log_rust_post_call,
+                    python_fallback=lambda: self.async_completion(
+                        model=model,
+                        messages=messages,
+                        api_base=proxy_endpoint_url,
+                        model_response=model_response,
+                        encoding=encoding,
+                        logging_obj=logging_obj,
+                        optional_params=optional_params,
+                        stream=stream,
+                        litellm_params=litellm_params,
+                        logger_fn=logger_fn,
+                        headers=headers,
+                        timeout=timeout,
+                        client=client,
+                        credentials=credentials,
+                        api_key=api_key,
+                        skip_pre_call_logging=True,
+                    ),
+                )
+            rust_response: Final = rust_chat_completions_bridge.chat_completions(
+                model=model,
+                messages=messages,
+                optional_params=rust_optional_params,
+                model_response=model_response,
+                api_key=api_key,
+                api_base=proxy_endpoint_url,
+                custom_llm_provider="bedrock",
+                extra_headers=headers,
+                timeout=timeout,
+                on_response=log_rust_post_call,
+            )
+            if rust_response is not None:
+                return rust_response
+
         ### ROUTING (ASYNC, STREAMING, SYNC)
         if acompletion:
             if isinstance(client, HTTPHandler):
@@ -420,15 +515,21 @@ class BedrockConverseLLM(BaseAWSLLM):
         )
 
         ## LOGGING
-        logging_obj.pre_call(
-            input=messages,
-            api_key="",
-            additional_args={
-                "complete_input_dict": data,
-                "api_base": proxy_endpoint_url,
-                "headers": prepped.headers,
-            },
-        )
+        # Reaching here with `serves_via_rust` set means the synchronous Rust
+        # attempt declined at call time, before the provider was called, and
+        # already logged this request. That is the same attempt continuing.
+        # The asynchronous branch above returns before this point, and hands
+        # its own fallback `skip_pre_call_logging=True` for the same reason.
+        if not serves_via_rust:
+            logging_obj.pre_call(
+                input=messages,
+                api_key="",
+                additional_args={
+                    "complete_input_dict": data,
+                    "api_base": proxy_endpoint_url,
+                    "headers": prepped.headers,
+                },
+            )
         if client is None or isinstance(client, AsyncHTTPHandler):
             _params: Final = {}
             if timeout is not None:

litellm/llms/custom_httpx/container_handler.py

@@ -39,6 +39,10 @@ RESPONSE_TYPES: Final[dict[str, type]] = {
     "DeleteContainerFileResponse": DeleteContainerFileResponse,
 }
 
+ContainerEndpointResponse = (
+    ContainerFileListResponse | ContainerFileObject | DeleteContainerFileResponse | bytes | dict[str, object]
+)
+
 
 def _load_endpoints_config() -> dict:
     """Load the endpoints configuration from JSON file."""
@@ -101,6 +105,51 @@ def _build_query_params(
     return params
 
 
+def _error_message_from_response(response: httpx.Response) -> str:
+    try:
+        body: Final = response.json()
+    except ValueError:
+        return response.text
+
+    if isinstance(body, dict) and isinstance(body.get("error"), dict):
+        message: Final = body["error"].get("message")
+        if isinstance(message, str):
+            return message
+
+    return response.text
+
+
+def _transform_response(
+    response: httpx.Response,
+    returns_binary: bool,
+    response_type_name: str,
+) -> ContainerEndpointResponse:
+    from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+    if httpx.codes.is_error(response.status_code):
+        raise BaseLLMException(
+            status_code=response.status_code,
+            message=_error_message_from_response(response),
+            headers=dict(response.headers),
+        )
+
+    if returns_binary:
+        return response.content
+
+    response_json: Final = response.json()
+    if "error" in response_json:
+        raise BaseLLMException(
+            status_code=response.status_code,
+            message=response_json.get("error", {}).get("message", str(response_json)),
+            headers=dict(response.headers),
+        )
+
+    response_type: Final = RESPONSE_TYPES.get(response_type_name)
+    if response_type:
+        return response_type(**response_json)
+    return response_json
+
+
 def _prepare_multipart_file_upload(
     file: Any,
     headers: dict[str, Any],
@@ -270,27 +319,11 @@ class GenericContainerHandler:
             else:
                 raise ValueError(f"Unsupported HTTP method: {method}")
 
-            # For binary responses, return raw content
-            if returns_binary:
-                return response.content
-
-            # Check for error response
-            response_json: Final = response.json()
-            if "error" in response_json:
-                from litellm.llms.base_llm.chat.transformation import BaseLLMException
-
-                error_msg: Final = response_json.get("error", {}).get("message", str(response_json))
-                raise BaseLLMException(
-                    status_code=response.status_code,
-                    message=error_msg,
-                    headers=dict(response.headers),
-                )
-
-            # Parse response
-            response_type: Final = RESPONSE_TYPES.get(endpoint_config["response_type"])
-            if response_type:
-                return response_type(**response_json)
-            return response_json
+            return _transform_response(
+                response=response,
+                returns_binary=returns_binary,
+                response_type_name=endpoint_config["response_type"],
+            )
 
         except Exception as e:
             raise e
@@ -378,27 +411,11 @@ class GenericContainerHandler:
             else:
                 raise ValueError(f"Unsupported HTTP method: {method}")
 
-            # For binary responses, return raw content
-            if returns_binary:
-                return response.content
-
-            # Check for error response
-            response_json: Final = response.json()
-            if "error" in response_json:
-                from litellm.llms.base_llm.chat.transformation import BaseLLMException
-
-                error_msg: Final = response_json.get("error", {}).get("message", str(response_json))
-                raise BaseLLMException(
-                    status_code=response.status_code,
-                    message=error_msg,
-                    headers=dict(response.headers),
-                )
-
-            # Parse response
-            response_type: Final = RESPONSE_TYPES.get(endpoint_config["response_type"])
-            if response_type:
-                return response_type(**response_json)
-            return response_json
+            return _transform_response(
+                response=response,
+                returns_binary=returns_binary,
+                response_type_name=endpoint_config["response_type"],
+            )
 
         except Exception as e:
             raise e

litellm/llms/fal_ai/cost_calculator.py

@@ -1,25 +1,75 @@
-from typing import Any, Final
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
 
 import litellm
 from litellm.types.utils import ImageResponse
 
+FAL_KEYED_PRICING_DEFAULT_QUALITY: Final[str] = "high"
+FAL_TEXT_TO_IMAGE_DEFAULT_SIZE: Final[str] = "1024-x-768"
+FAL_NAMED_IMAGE_SIZES: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "square_hd": "1024-x-1024",
+        "square": "512-x-512",
+        "portrait_4_3": "768-x-1024",
+        "portrait_16_9": "576-x-1024",
+        "landscape_4_3": "1024-x-768",
+        "landscape_16_9": "1024-x-576",
+    }
+)
+
+
+def _keyed_size(model: str, optional_params: Mapping[str, object]) -> str | None:
+    image_size: Final = optional_params.get("image_size")
+    if image_size is None:
+        return None if model.endswith("/edit") else FAL_TEXT_TO_IMAGE_DEFAULT_SIZE
+    if isinstance(image_size, Mapping):
+        width: Final = image_size.get("width")
+        height: Final = image_size.get("height")
+        if isinstance(width, int) and isinstance(height, int):
+            return f"{width}-x-{height}"
+        return None
+    if isinstance(image_size, str):
+        return FAL_NAMED_IMAGE_SIZES.get(image_size)
+    return None
+
+
+def _keyed_cost_per_image(model: str, optional_params: Mapping[str, object] | None) -> float | None:
+    if optional_params is None:
+        return None
+    size: Final = _keyed_size(model=model, optional_params=optional_params)
+    if size is None:
+        return None
+    raw_quality: Final = optional_params.get("quality")
+    quality: Final = (
+        raw_quality if isinstance(raw_quality, str) and raw_quality != "auto" else FAL_KEYED_PRICING_DEFAULT_QUALITY
+    )
+    keyed_entry: Final = litellm.model_cost.get(f"fal_ai/{quality}/{size}/{model}")
+    if keyed_entry is None:
+        return None
+    keyed_cost: Final = keyed_entry.get("output_cost_per_image")
+    return float(keyed_cost) if isinstance(keyed_cost, (int, float)) else None
+
 
 def cost_calculator(
     model: str,
-    image_response: Any,
+    image_response: object,
+    optional_params: Mapping[str, object] | None = None,
 ) -> float:
     """
     fal.ai image generation cost calculator
     """
+    if not isinstance(image_response, ImageResponse):
+        raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
+    # the proxy cost path passes the provider-prefixed model name
+    model = model.removeprefix(f"{litellm.LlmProviders.FAL_AI.value}/")
+    num_images: Final[int] = len(image_response.data) if image_response.data else 0
+    keyed_cost_per_image: Final = _keyed_cost_per_image(model=model, optional_params=optional_params)
+    if keyed_cost_per_image is not None:
+        return keyed_cost_per_image * num_images
     _model_info: Final = litellm.get_model_info(
         model=model,
         custom_llm_provider=litellm.LlmProviders.FAL_AI.value,
     )
     output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0
-    num_images: int = 0
-    if isinstance(image_response, ImageResponse):
-        if image_response.data:
-            num_images = len(image_response.data)
-        return output_cost_per_image * num_images
-    else:
-        raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
+    return output_cost_per_image * num_images

litellm/llms/fal_ai/image_generation/__init__.py

@@ -12,6 +12,7 @@ from .bytedance_transformation import (
 from .flux_pro_v11_transformation import FalAIFluxProV11Config
 from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig
 from .flux_schnell_transformation import FalAIFluxSchnellConfig
+from .gpt_image_2_transformation import FalAIGPTImage2Config
 from .ideogram_v3_transformation import FalAIIdeogramV3Config
 from .imagen4_transformation import FalAIImagen4Config
 from .nano_banana_transformation import FalAINanoBananaConfig
@@ -27,6 +28,7 @@ __all__ = [
     "FalAIFluxProV11Config",
     "FalAIFluxProV11UltraConfig",
     "FalAIFluxSchnellConfig",
+    "FalAIGPTImage2Config",
     "FalAIIdeogramV3Config",
     "FalAIImageGenerationConfig",
     "FalAIImagen4Config",
@@ -49,7 +51,9 @@ def get_fal_ai_image_generation_config(model: str) -> BaseImageGenerationConfig:
     model_lower: Final = model.lower()
 
     # Map model names to their corresponding configuration classes
-    if "nano-banana" in model_lower or "gemini-25-flash-image" in model_lower:
+    if "gpt-image-2" in model_lower:
+        return FalAIGPTImage2Config()
+    elif "nano-banana" in model_lower or "gemini-25-flash-image" in model_lower:
         return FalAINanoBananaConfig()
     elif "imagen4" in model_lower or "imagen-4" in model_lower:
         return FalAIImagen4Config()

litellm/llms/fal_ai/image_generation/gpt_image_2_transformation.py

@@ -0,0 +1,124 @@
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
+
+from .transformation import FalAIBaseConfig
+
+
+class FalAIImageSize(TypedDict):
+    width: ReadOnly[int]
+    height: ReadOnly[int]
+
+
+SUPPORTED_OPENAI_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = (
+    "n",
+    "output_format",
+    "quality",
+    "response_format",
+    "size",
+)
+
+
+class FalAIGPTImage2Config(FalAIBaseConfig):
+    """
+    Configuration for OpenAI's GPT Image 2 served through Fal AI.
+
+    Model endpoints:
+    - openai/gpt-image-2 (text-to-image)
+    - openai/gpt-image-2/edit (editing, with optional mask)
+
+    Documentation: https://fal.ai/models/openai/gpt-image-2/api
+    """
+
+    MODEL_PREFIX: Final[str] = "openai/"
+    SUPPORTED_QUALITIES: Final[frozenset[str]] = frozenset({"auto", "low", "medium", "high"})
+    OPENAI_QUALITY_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"hd": "high", "standard": "medium"})
+    PARAM_TRANSLATION: Final[Mapping[str, str]] = MappingProxyType(
+        {
+            "n": "num_images",
+            "size": "image_size",
+            "quality": "quality",
+            "output_format": "output_format",
+        }
+    )
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        base_url: Final[str] = (api_base or get_secret_str("FAL_AI_API_BASE") or self.DEFAULT_BASE_URL).rstrip("/")
+        endpoint: Final[str] = model if model.startswith(self.MODEL_PREFIX) else f"{self.MODEL_PREFIX}{model}"
+        return f"{base_url}/{endpoint}"
+
+    def get_supported_openai_params(  # mutable-ok: base class contract returns a list
+        self, model: str
+    ) -> list[OpenAIImageGenerationOptionalParams]:
+        return list(SUPPORTED_OPENAI_PARAMS)  # mutable-ok: base class contract returns a list
+
+    def map_openai_params(  # mutable-ok: base class contract returns a dict
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        unsupported_params: Final = tuple(
+            key for key in non_default_params if key not in SUPPORTED_OPENAI_PARAMS and key not in optional_params
+        )
+        if unsupported_params and not drop_params:
+            raise ValueError(
+                f"Parameters {unsupported_params} are not supported for model {model}. "
+                f"Supported parameters are {SUPPORTED_OPENAI_PARAMS}. "
+                "Set drop_params=True to drop unsupported parameters."
+            )
+        translated_params: Final[Mapping[str, object]] = MappingProxyType(
+            {
+                self.PARAM_TRANSLATION[key]: self._translate_value(key, value)
+                for key, value in non_default_params.items()
+                if key in self.PARAM_TRANSLATION and self.PARAM_TRANSLATION[key] not in optional_params
+            }
+        )
+        return {**optional_params, **translated_params}  # mutable-ok: base class contract returns a dict
+
+    def _translate_value(self, key: str, value: object) -> object:
+        if key == "size":
+            return self._map_image_size(value)
+        if key == "quality":
+            return self._map_quality(value)
+        return value
+
+    def _map_image_size(self, size: object) -> object:
+        if not isinstance(size, str) or size == "auto":
+            return size
+        try:
+            width, height = (int(part) for part in size.lower().split("x"))
+        except ValueError:
+            return size
+        image_size: Final[FalAIImageSize] = {"width": width, "height": height}
+        return image_size
+
+    def _map_quality(self, quality: object) -> object:
+        if not isinstance(quality, str):
+            return quality
+        normalized: Final[str] = self.OPENAI_QUALITY_ALIASES.get(quality, quality)
+        return normalized if normalized in self.SUPPORTED_QUALITIES else "auto"
+
+    def transform_image_generation_request(  # mutable-ok: base class contract returns a dict
+        self,
+        model: str,
+        prompt: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        headers: Mapping[str, str],
+    ) -> dict:
+        return {"prompt": prompt, **optional_params}  # mutable-ok: base class contract returns a dict

litellm/llms/openai_like/json_loader.py

@@ -65,6 +65,11 @@ class JSONProviderRegistry:
         """Check if a provider is defined via JSON"""
         return slug in cls._providers
 
+    @classmethod
+    def get_by_base_url(cls, base_url: str) -> SimpleProviderConfig | None:
+        """Get a provider configuration by its default base url"""
+        return next((provider for provider in cls._providers.values() if provider.base_url == base_url), None)
+
     @classmethod
     def supports_responses_api(cls, slug: str) -> bool:
         """Check if a JSON provider supports the Responses API"""

litellm/llms/openai_like/providers.json

@@ -175,6 +175,11 @@
     "base_class": "openai_gpt",
     "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
   },
+  "cognition": {
+    "base_url": "https://api.cognition.ai/v1",
+    "api_key_env": "COGNITION_API_KEY",
+    "api_base_env": "COGNITION_API_BASE"
+  },
   "pinstripes": {
     "base_url": "https://pinstripes.io/v1",
     "api_key_env": "PINSTRIPES_API_KEY",

litellm/llms/perplexity/cost_calculator.py

@@ -21,14 +21,19 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
         Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
     """
     ## USE PRE-CALCULATED COST FROM PERPLEXITY IF AVAILABLE
-    ## Perplexity returns accurate cost in usage.cost.total_cost including request fees
+    ## Perplexity returns accurate cost in usage.cost.total_cost including request fees.
+    ## By the time it reaches here, ResponseAPIUsage.parse_cost has already flattened
+    ## that dict down to a float, so both shapes must be accepted.
     cost_info: Final = getattr(usage, "cost", None)
-    if cost_info is not None and isinstance(cost_info, dict):
-        total_cost: Final = cost_info.get("total_cost")
-        if total_cost is not None:
-            # Return total cost as completion_cost (prompt_cost=0) since Perplexity
-            # doesn't break down by input/output in their cost object
-            return (0.0, float(total_cost))
+    total_cost: float | None = None
+    if isinstance(cost_info, dict):
+        total_cost = cost_info.get("total_cost")
+    elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool):
+        total_cost = float(cost_info)
+    if total_cost is not None:
+        # Return total cost as completion_cost (prompt_cost=0) since Perplexity
+        # doesn't break down by input/output in their cost object
+        return (0.0, float(total_cost))
 
     ## FALLBACK: Calculate cost manually if Perplexity doesn't provide it
     ## GET MODEL INFO

litellm/llms/vertex_ai/gemini/transformation.py

@@ -645,10 +645,9 @@ def _collect_tool_call_thought_signatures(
     the text part as well would send two copies and double-bill the previous
     turn's reasoning tokens on gemini-3 and newer models.
 
-    Detection deliberately calls _get_thought_signature_from_tool without the
-    model argument: with a gemini-3 model that helper synthesizes a dummy
-    signature for unsigned tool calls, which must not suppress a real
-    text-part signature (e.g. replaying gemini-2.5 history to a newer model).
+    Only real signatures count here; a synthesized placeholder must not
+    suppress a genuine text-part signature (e.g. replaying gemini-2.5 history
+    to a newer model).
     """
     signatures: tuple[str, ...] = ()
 

litellm/main.py

@@ -75,7 +75,7 @@ from litellm.litellm_core_utils.chat_completion_agentic_loop import (
 from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.get_litellm_params import (
-    AWS_CREDENTIAL_KWARGS_KEYS,
+    FORWARDED_KWARGS_KEYS,
     OPTIONAL_KWARGS_KEYS,
 )
 from litellm.litellm_core_utils.get_provider_specific_headers import (
@@ -5451,7 +5451,7 @@ def completion(
             tpm=kwargs.get("tpm"),
             rpm=kwargs.get("rpm"),
             use_xai_oauth=kwargs.get("use_xai_oauth", False),
-            **{key: kwargs[key] for key in AWS_CREDENTIAL_KWARGS_KEYS if key in kwargs},
+            **{key: kwargs[key] for key in FORWARDED_KWARGS_KEYS if key in kwargs},
         )
         cast(LiteLLMLoggingObj, logging).update_environment_variables(
             model=model,
@@ -5974,7 +5974,7 @@ def embedding(
     # Optional params
     dimensions: int | None = None,
     encoding_format: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float = 600,  # default to 10 minutes
     # set api_base, api_version, api_key
     api_base: str | None = None,
     api_version: str | None = None,
@@ -6000,7 +6000,7 @@ def embedding(
     # Optional params
     dimensions: int | None = None,
     encoding_format: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float = 600,  # default to 10 minutes
     # set api_base, api_version, api_key
     api_base: str | None = None,
     api_version: str | None = None,
@@ -6027,7 +6027,7 @@ def embedding(
     # Optional params
     dimensions: int | None = None,
     encoding_format: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float = 600,  # default to 10 minutes
     # set api_base, api_version, api_key
     api_base: str | None = None,
     api_version: str | None = None,