Merge pull request #47519 from BerriAI/patch_27570705

chore(release): v1.105.2 patch

.circleci/scripts/run_integration.sh

@@ -169,7 +169,7 @@ start_proxy() {
     INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
     LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
     LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
-    LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True "${cost_map_env[@]}" \
+    LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
     AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
     "${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
     --host 127.0.0.1 --port "$port" --num_workers 1 --telemetry False \

.github/workflows/test-unit.yml

@@ -61,7 +61,7 @@ jobs:
 
           - shard: core-utils
             artifact-name: core-utils
-            test-path: ""
+            test-path: tests/unit/decisions
             unit-flag: core-utils
             workers: 2
             reruns: 1
@@ -141,6 +141,7 @@ jobs:
             artifact-name: proxy-endpoints
             test-path: >-
               tests/unit/proxy/analytics_endpoints
+              tests/unit/proxy/decisions_endpoints
               tests/unit/proxy/management_endpoints
               tests/unit/proxy/list_api
               tests/unit/proxy/memory

Dockerfile

@@ -9,8 +9,8 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
 # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
-ARG PGBOUNCER_VERSION=1.25.2
+ARG PGBOUNCER_VERSION=1.26.0
-ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
+ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
 
 FROM $UV_IMAGE AS uvbin
 

README.md

@@ -390,11 +390,13 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) |  |  |  |  |  |  |  |  |  |  |
 | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ |  |
 | [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Typesafe Decisions API (`typesafe`)](https://docs.litellm.ai/docs/providers) |  |  |  |  |  |  |  |  |  |  |
 | [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

ci_cd/cost_map_guard.py

@@ -14,8 +14,9 @@ import argparse
 import json
 import subprocess
 import sys
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
+from types import MappingProxyType
 from typing import Final
 
 from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
@@ -25,6 +26,7 @@ BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
 SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
 GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
 BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
 
 CostMap = dict[str, object]
 
@@ -51,7 +53,31 @@ def _rendered_schema(cost_map: CostMap) -> str:
         return str(error)
 
 
-def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]:
+def _evaluation_output_price_failure(
+    key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
+) -> str | None:
+    if not isinstance(entry, dict) or entry.get("mode") != "evaluation":
+        return None
+    output_cost: Final = entry.get("output_cost_per_token")
+    if (
+        not isinstance(output_cost, (int, float))
+        or isinstance(output_cost, bool)
+        or output_cost <= 0
+        or key in evaluation_output_price_sources
+    ):
+        return None
+    return (
+        f"{key}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor "
+        "prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)"
+    )
+
+
+def _file_failures(
+    head: Snapshot,
+    head_map: CostMap,
+    *,
+    evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
+) -> tuple[str, ...]:
     schema_text: Final = _rendered_schema(head_map)
     if not schema_text.startswith("{"):
         return (schema_text,)
@@ -75,6 +101,16 @@ def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]:
             f"{COST_MAP_PATH} does not validate against its schema: {error}"
             for error in validation_errors(head_map, json.loads(schema_text))[:20]
         ),
+        *(
+            failure
+            for key, entry in head_map.items()
+            if (
+                failure := _evaluation_output_price_failure(
+                    key, entry, evaluation_output_price_sources
+                )
+            )
+            is not None
+        ),
     )
 
 
@@ -121,13 +157,25 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
     return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
 
 
-def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
+def guard_failures(
+    base: Snapshot,
+    head: Snapshot,
+    changed_files: Sequence[str],
+    bot: bool,
+    *,
+    evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
+) -> tuple[str, ...]:
     if not bot and not touches_cost_map(changed_files):
         return ()
     head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
     if isinstance(head_map, str):
         return (head_map,)
-    return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
+    return (
+        *_file_failures(
+            head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
+        ),
+        *(_bot_failures(base, head_map, changed_files) if bot else ()),
+    )
 
 
 def _git(*args: str) -> str | None:

ci_cd/generate_model_prices_schema.py

@@ -19,6 +19,7 @@ NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
 NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
 BOOLEAN: JsonSchema = {"type": "boolean"}
 STRING: JsonSchema = {"type": "string"}
+NON_BLANK_STRING: Final[JsonSchema] = {"type": "string", "pattern": r"\S"}
 TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"}
 WEEKDAY_PATTERN: Final = (
     r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
@@ -103,6 +104,16 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
         "type": "object",
         "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
     },
+    "token_count_model": {
+        "type": "object",
+        "required": ["model"],
+        "additionalProperties": False,
+        "properties": {
+            "model": NON_BLANK_STRING,
+            "region": NON_BLANK_STRING,
+        },
+        "description": "Model that counts tokens for this one, and the AWS region its count API runs in.",
+    },
 }
 
 ARRAY_KEYS: dict[str, JsonSchema] = {
@@ -189,6 +200,10 @@ INTEGER_KEYS: dict[str, JsonSchema] = {
         **NONNEG_INTEGER,
         "description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
     },
+    "token_count_framing": {
+        **NONNEG_INTEGER,
+        "description": "Tokens added once per request to a local count estimate, for the provider's chat framing.",
+    },
     "tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
     "rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
 }
@@ -244,6 +259,11 @@ def cost_schema(key: str) -> JsonSchema:
 
 def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
     return {
+        "token_count_tokenizer": {
+            "type": "string",
+            "description": "Tokenizer litellm counts this model's requests with when the provider cannot count them.",
+            "enum": ["kimi_k3"],
+        },
         "litellm_provider": {
             "type": "string",
             "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",

docker/Dockerfile.database

@@ -9,8 +9,8 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
 # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
-ARG PGBOUNCER_VERSION=1.25.2
+ARG PGBOUNCER_VERSION=1.26.0
-ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
+ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
 
 FROM $UV_IMAGE AS uvbin
 

docker/Dockerfile.non_root

@@ -7,8 +7,8 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
 # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
-ARG PGBOUNCER_VERSION=1.25.2
+ARG PGBOUNCER_VERSION=1.26.0
-ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
+ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
 
 FROM $UV_IMAGE AS uvbin
 

gateway/Dockerfile

@@ -2,8 +2,8 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9a
 ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
-ARG PGBOUNCER_VERSION=1.25.2
+ARG PGBOUNCER_VERSION=1.26.0
-ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
+ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
 
 FROM $UV_IMAGE AS uvbin
 

gateway/routes/allowlist.py

@@ -1,7 +1,7 @@
 """Path allowlist for the gateway component.
 
 The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
-audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
+audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image,
 responses, vector stores, passthrough providers, realtime websockets, MCP
 tool-call endpoints, and operational endpoints (/health, /metrics, and the
 /debug/memory/summary read of the serving worker's RSS).
@@ -60,6 +60,10 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/v1/rerank",
     "/v2/rerank",
     "/rerank",
+    "/v1/decisions",
+    "/decisions",
+    "/v1/systemone",
+    "/systemone",
     "/v1/ocr",
     "/ocr",
     "/v1/rag/",

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -78,6 +78,23 @@ class _InvalidIndex:
     table_size: str
 
 MAX_MIGRATE_DEPLOY_ATTEMPTS = 4
+LIBPQ_URL_PARAMS: Final = frozenset(
+    {
+        "sslmode",
+        "sslcert",
+        "sslkey",
+        "sslrootcert",
+        "sslpassword",
+        "application_name",
+        "connect_timeout",
+        "client_encoding",
+        "options",
+        "service",
+        "gssencmode",
+        "krbsrvname",
+        "target_session_attrs",
+    }
+)
 
 
 @dataclass(frozen=True)
@@ -336,9 +353,8 @@ class ProxyExtrasDBManager:
             pass
 
     @staticmethod
-    def _failed_migration_logs(migration_name: str) -> Optional[str]:
+    def _read_migration_ledger(query: str, params: tuple[str, ...]) -> "tuple[object, ...] | None":
-        """Return failed migration logs, or None if the ledger is unavailable."""
+        database_url: Final = os.getenv("DATABASE_URL")
-        database_url = os.getenv("DATABASE_URL")
         if not database_url:
             return None
 
@@ -347,28 +363,37 @@ class ProxyExtrasDBManager:
         except ImportError:
             return None
 
-        cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
+        cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
-        ledger_table = psycopg.sql.SQL("{}.{}").format(
+        ledger_table: Final = psycopg.sql.SQL("{}.{}").format(
-            psycopg.sql.Identifier(
+            psycopg.sql.Identifier(ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"),
-                ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
-            ),
             psycopg.sql.Identifier("_prisma_migrations"),
         )
         try:
-            with psycopg.connect(
+            with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
-                cleaned_url, connect_timeout=10, autocommit=True
+                row: Final = conn.execute(psycopg.sql.SQL(query).format(ledger_table), params).fetchone()
-            ) as conn:
-                row = conn.execute(
-                    psycopg.sql.SQL(
-                        "SELECT logs FROM {} "
-                        "WHERE migration_name = %s AND finished_at IS NULL "
-                        "AND rolled_back_at IS NULL"
-                    ).format(ledger_table),
-                    (migration_name,),
-                ).fetchone()
         except (psycopg.OperationalError, psycopg.DatabaseError):
             return None
-        return (row[0] or "") if row else ""
+        return tuple(row) if row is not None else ()
+
+    @staticmethod
+    def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]:
+        row: Final = ProxyExtrasDBManager._read_migration_ledger(
+            "SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
+            "AND finished_at IS NULL AND rolled_back_at IS NULL",
+            (migration_name, started_at),
+        )
+        if row is None:
+            return None
+        return row[0] if row and isinstance(row[0], str) else ""
+
+    @staticmethod
+    def _failed_migration_recovered(migration_name: str, started_at: str) -> bool:
+        row: Final = ProxyExtrasDBManager._read_migration_ledger(
+            "SELECT 1 FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
+            "AND (finished_at IS NOT NULL OR rolled_back_at IS NOT NULL)",
+            (migration_name, started_at),
+        )
+        return bool(row)
 
     @staticmethod
     def _resolve_specific_migration(migration_name: str):
@@ -689,30 +714,43 @@ class ProxyExtrasDBManager:
 
     @staticmethod
     def _strip_prisma_query_params(url: str) -> str:
-        """Remove Prisma-specific query params (connection_limit, pool_timeout,
+        """Rewrite a Prisma-dialect URL for libpq: drop the Prisma-only params
-        schema, etc.) from DATABASE_URL so psycopg can parse it."""
+        (connection_limit, pool_timeout, schema, pgbouncer, sslaccept, ...) and
+        translate Prisma's TLS params back, since libpq reads ``sslcert`` as a
+        client certificate where Prisma reads it as the CA."""
         from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
 
-        parsed = urlparse(url)
+        parsed: Final = urlparse(url)
         if not parsed.query:
             return url
-        libpq_params = {
+        pairs: Final = tuple(parse_qsl(parsed.query))
-            "sslmode",
+        kept: Final = tuple((k, v) for k, v in pairs if k in LIBPQ_URL_PARAMS)
-            "sslcert",
+        sslaccept: Final = next((v for k, v in pairs if k == "sslaccept"), None)
-            "sslkey",
+        libpq_pairs: Final = ProxyExtrasDBManager._libpq_tls_params(kept, sslaccept)
-            "sslrootcert",
+        return urlunparse(parsed._replace(query=urlencode(libpq_pairs, quote_via=quote)))
-            "sslpassword",
+
-            "application_name",
+    @staticmethod
-            "connect_timeout",
+    def _libpq_tls_params(
-            "client_encoding",
+        pairs: "tuple[tuple[str, str], ...]", sslaccept: "str | None"
-            "options",
+    ) -> "tuple[tuple[str, str], ...]":
-            "service",
+        """Undo ``translate_libpq_ssl_params``. Prisma's ``sslcert`` is the CA and
-            "gssencmode",
+        ``sslaccept=strict`` checks chain and hostname, which libpq only does in
-            "krbsrvname",
+        ``sslmode=verify-full``, so strict becomes ``sslrootcert`` plus
-            "target_session_attrs",
+        ``verify-full`` whatever ``sslmode`` said (``disable`` stays off). Prisma
-        }
+        defaults an absent ``sslaccept`` to ``accept_invalid_certs`` and anything
-        kept = [(k, v) for k, v in parse_qsl(parsed.query) if k in libpq_params]
+        else to strict. Without strict it checks nothing, so the CA is dropped and
-        return urlunparse(parsed._replace(query=urlencode(kept, quote_via=quote)))
+        ``sslmode`` is kept as is: libpq only verifies when a root cert is present.
+        A URL that also carries ``sslkey`` is libpq's own client-certificate form
+        and is kept."""
+        keys: Final = frozenset(k for k, _ in pairs)
+        if "sslcert" not in keys or "sslkey" in keys:
+            return pairs
+        sslmode: Final = next((v for k, v in pairs if k == "sslmode"), None)
+        rest: Final = tuple((k, v) for k, v in pairs if k not in ("sslcert", "sslmode"))
+        if sslaccept in (None, "accept_invalid_certs") or sslmode == "disable":
+            return rest if sslmode is None else rest + (("sslmode", sslmode),)
+        root_cert: Final = tuple(("sslrootcert", v) for k, v in pairs if k == "sslcert" and "sslrootcert" not in keys)
+        return rest + root_cert + (("sslmode", "verify-full"),)
 
     @staticmethod
     def _warn_if_db_ahead_of_head(migrations_dir: str) -> None:
@@ -1072,6 +1110,11 @@ class ProxyExtrasDBManager:
             return match.group(1) if match else None
         return None
 
+    @staticmethod
+    def _v2_failed_migration_started_at(stderr: str, migration_name: str) -> "str | None":
+        match: Final = re.search(rf"`{re.escape(migration_name)}` migration started at ([^\r\n]+?) failed", stderr)
+        return match.group(1) if match else None
+
     @staticmethod
     def _v2_roll_back_migration_best_effort(migration_name: str) -> None:
         from litellm_proxy_extras.migration_lock import migration_environment
@@ -1100,8 +1143,11 @@ class ProxyExtrasDBManager:
 
         if "P3009" in stderr:
             migration_name = ProxyExtrasDBManager._v2_failed_migration_name(stderr)
-            if migration_name:
+            started_at: Final = (
-                ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name)
+                ProxyExtrasDBManager._v2_failed_migration_started_at(stderr, migration_name) if migration_name else None
+            )
+            if migration_name and started_at:
+                ledger_logs: Final = ProxyExtrasDBManager._failed_migration_logs(migration_name, started_at)
                 if ledger_logs and _MIGRATION_DEADLOCK_MARKER in ledger_logs:
                     logger.info(
                         "Migration %s failed in a concurrent migrate deploy "
@@ -1110,6 +1156,14 @@ class ProxyExtrasDBManager:
                     )
                     ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)
                     return budget.spend()
+                if ProxyExtrasDBManager._failed_migration_recovered(migration_name, started_at):
+                    logger.info(
+                        "Migration %s started at %s was already rolled back or completed by a concurrent "
+                        "migrate deploy, retrying",
+                        migration_name,
+                        started_at,
+                    )
+                    return budget.spend()
             raise RuntimeError(
                 "Migration completion could not be verified. LiteLLM startup has stopped.\n\n"
                 f"Prisma migration history (migration name and start time):\n{stderr}\n\n"

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.104"
+version = "0.4.105"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.104"
+version = "0.4.105"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/crates/model-catalog/src/model_info.rs

@@ -467,6 +467,10 @@ pub struct ModelInfo {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub supports_audio_output: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub supports_bedrock_runtime_chat_completions_response_format: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub supports_bedrock_runtime_chat_completions_tools_with_reasoning: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub supports_computer_use: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub supports_embedding_image_input: Option<bool>,

litellm/__init__.py

@@ -1473,6 +1473,8 @@ from .embeddings.dispatch import *
 from .rust_bridge import rust
 from .rag.main import *
 from .sandbox.main import *
+from .decisions.main import *
+from .systemone.main import *
 from .search.main import *
 from .realtime_api.main import (
     _arealtime,
@@ -1653,6 +1655,30 @@ if TYPE_CHECKING:
     from .llms.hosted_vllm.rerank.transformation import (
         HostedVLLMRerankConfig as HostedVLLMRerankConfig,
     )
+    from .llms.perplexity.decisions.transformation import (
+        PerplexityDecisionsConfig as PerplexityDecisionsConfig,
+    )
+    from .llms.typesafe.decisions.transformation import (
+        TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
+    )
+    from .llms.openrouter.decisions.transformation import (
+        OpenRouterDecisionsConfig as OpenRouterDecisionsConfig,
+    )
+    from .llms.cloudflare.decisions.transformation import (
+        CloudflareDecisionsConfig as CloudflareDecisionsConfig,
+    )
+    from .llms.strands_decider.decisions.transformation import (
+        StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig,
+    )
+    from .llms.hosted_vllm.decisions.transformation import (
+        HostedVLLMDecisionsConfig as HostedVLLMDecisionsConfig,
+    )
+    from .llms.openai.decisions.transformation import (
+        OpenAIDecisionsConfig as OpenAIDecisionsConfig,
+    )
+    from .llms.azure_ai.decisions.transformation import (
+        AzureAIDecisionsConfig as AzureAIDecisionsConfig,
+    )
     from .llms.nvidia_nim.rerank.transformation import (
         NvidiaNimRerankConfig as NvidiaNimRerankConfig,
     )
@@ -1780,6 +1806,9 @@ if TYPE_CHECKING:
     from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
         AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig,
     )
+    from .llms.bedrock.chat.chat_completions.transformation import (
+        AmazonBedrockRuntimeChatCompletionsConfig as AmazonBedrockRuntimeChatCompletionsConfig,
+    )
     from .llms.bedrock.image_generation.amazon_stability1_transformation import (
         AmazonStabilityConfig as AmazonStabilityConfig,
     )

litellm/_lazy_imports_registry.py

@@ -153,6 +153,14 @@ LLM_CONFIG_NAMES: Final = (
     "JinaAIRerankConfig",
     "DeepinfraRerankConfig",
     "HostedVLLMRerankConfig",
+    "PerplexityDecisionsConfig",
+    "TypeSafeDecisionsConfig",
+    "OpenRouterDecisionsConfig",
+    "CloudflareDecisionsConfig",
+    "StrandsDeciderDecisionsConfig",
+    "HostedVLLMDecisionsConfig",
+    "OpenAIDecisionsConfig",
+    "AzureAIDecisionsConfig",
     "NvidiaNimRerankConfig",
     "NvidiaNimRankingConfig",
     "VertexAIRerankConfig",
@@ -206,6 +214,7 @@ LLM_CONFIG_NAMES: Final = (
     "AmazonTwelveLabsPegasusConfig",
     "AmazonInvokeConfig",
     "AmazonBedrockOpenAIConfig",
+    "AmazonBedrockRuntimeChatCompletionsConfig",
     "AmazonStabilityConfig",
     "AmazonStability3Config",
     "AmazonNovaCanvasConfig",
@@ -695,6 +704,20 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         ".llms.hosted_vllm.rerank.transformation",
         "HostedVLLMRerankConfig",
     ),
+    "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
+    "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
+    "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
+    "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),
+    "StrandsDeciderDecisionsConfig": (
+        ".llms.strands_decider.decisions.transformation",
+        "StrandsDeciderDecisionsConfig",
+    ),
+    "HostedVLLMDecisionsConfig": (
+        ".llms.hosted_vllm.decisions.transformation",
+        "HostedVLLMDecisionsConfig",
+    ),
+    "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
+    "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
     "NvidiaNimRerankConfig": (
         ".llms.nvidia_nim.rerank.transformation",
         "NvidiaNimRerankConfig",
@@ -868,6 +891,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         ".llms.bedrock.chat.invoke_transformations.amazon_openai_transformation",
         "AmazonBedrockOpenAIConfig",
     ),
+    "AmazonBedrockRuntimeChatCompletionsConfig": (
+        ".llms.bedrock.chat.chat_completions.transformation",
+        "AmazonBedrockRuntimeChatCompletionsConfig",
+    ),
     "AmazonStabilityConfig": (
         ".llms.bedrock.image_generation.amazon_stability1_transformation",
         "AmazonStabilityConfig",

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -5,6 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 import json
 import os
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
+from itertools import accumulate, chain
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast
 
@@ -69,7 +70,7 @@ if TYPE_CHECKING:
         ChatCompletionToolReferenceObject,
         OpenAIMessageContentListBlock,
     )
-    from litellm.types.utils import Choices
+    from litellm.types.utils import Choices, Message
 
 
 _CHAT_COMPLETION_FIELDS: Final = frozenset((*ModelResponse.model_fields, "usage"))
@@ -187,13 +188,74 @@ def _reasoning_items_from_output_items(output_items: Sequence[object]) -> tuple[
 
 
 def _as_chat_reasoning_items(
-    reasoning_items: Sequence[_BuiltReasoningItem],
+    reasoning_items: Sequence[_BuiltReasoningItem | ChatCompletionReasoningItem],
 ) -> list[ChatCompletionReasoningItem] | None:
     if not reasoning_items:
         return None
     return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
 
 
+def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation:
+    shifted: Final = {
+        key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value
+        for key, value in annotation.items()
+    }
+    return cast(ChatCompletionAnnotation, shifted)
+
+
+def _message_annotations(message: "Message") -> Sequence[ChatCompletionAnnotation]:
+    annotations: Sequence[ChatCompletionAnnotation] | None = getattr(message, "annotations", None)
+    return annotations or ()
+
+
+def _message_reasoning_content(message: "Message") -> str | None:
+    reasoning_content: str | None = getattr(message, "reasoning_content", None)
+    return reasoning_content
+
+
+def _message_reasoning_items(message: "Message") -> Sequence[ChatCompletionReasoningItem]:
+    reasoning_items: Sequence[ChatCompletionReasoningItem] | None = getattr(message, "reasoning_items", None)
+    return reasoning_items or ()
+
+
+def _shifted_annotations(messages: Sequence["Message"], offsets: Sequence[int]) -> Iterator[ChatCompletionAnnotation]:
+    for message, offset in zip(messages, offsets):
+        for annotation in _message_annotations(message):
+            yield _shift_annotation(annotation, offset)
+
+
+def _fold_choices_into_one(
+    choices: Sequence["Choices"],
+    tool_calls: Sequence[Mapping[str, object]],
+    trailing_reasoning_content: str | None,
+    trailing_reasoning_item: _BuiltReasoningItem | None,
+) -> "Choices":
+    """Clients that only read choices[0] must still see every text and tool call."""
+    from litellm.types.utils import Choices, Message
+
+    messages: Final = tuple(choice.message for choice in choices)
+    texts: Final = tuple(str(message.content or "") for message in messages)
+    offsets: Final = (0, *accumulate(len(text) for text in texts))
+    annotations: Final = tuple(_shifted_annotations(messages, offsets))
+    reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
+    reasoning_items: Final = (
+        *chain.from_iterable(_message_reasoning_items(m) for m in messages),
+        *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
+    )
+    return Choices(
+        message=Message(
+            role=messages[0].role if messages else "assistant",
+            content="".join(texts) if messages else None,
+            annotations=list(annotations) or None,
+            tool_calls=list(tool_calls) or None,
+            reasoning_content=" ".join(value for value in reasoning_contents if value) or None,
+            reasoning_items=_as_chat_reasoning_items(reasoning_items),
+        ),
+        finish_reason="tool_calls" if tool_calls else "stop",
+        index=0,
+    )
+
+
 _ToolChoiceT = TypeVar("_ToolChoiceT")
 
 
@@ -784,22 +846,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             else:
                 pass  # don't fail request if item in list is not supported
 
-        # If we accumulated tool calls, create a single choice with all of them
+        if not choices and not accumulated_tool_calls:
-        if accumulated_tool_calls:
+            return choices
-            msg = Message(
+        return [
-                content=None,
+            _fold_choices_into_one(
+                choices=choices,
                 tool_calls=accumulated_tool_calls,
-                reasoning_content=reasoning_content,
+                trailing_reasoning_content=reasoning_content,
-                reasoning_items=cast(
+                trailing_reasoning_item=pending_reasoning_item,
-                    list[ChatCompletionReasoningItem] | None,
-                    ([pending_reasoning_item] if pending_reasoning_item is not None else None),
-                ),
             )
-            choices.append(Choices(message=msg, finish_reason="tool_calls", index=index))
+        ]
-            reasoning_content = None
-            pending_reasoning_item = None
-
-        return choices
 
     @staticmethod
     def _build_empty_incomplete_choice(

litellm/constants.py

@@ -266,6 +266,7 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int(
 
 # Provider-specific API base URLs
 XAI_API_BASE: Final = "https://api.x.ai/v1"
+PLACEHOLDER_API_KEY: Final = "fake-api-key"
 OPEN_SANDBOX_API_BASE_ENV_VAR: Final = "OPEN_SANDBOX_API_BASE"
 OPEN_SANDBOX_API_KEY_ENV_VAR: Final = "OPEN_SANDBOX_API_KEY"
 OPEN_SANDBOX_DEFAULT_TEMPLATE: Final = "opensandbox/code-interpreter:v1.1.0"

litellm/cost_calculator.py

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast
 
 from httpx import Response
 from pydantic import BaseModel
-from typing_extensions import ReadOnly, TypedDict
+from typing_extensions import ReadOnly, TypedDict, assert_never
 
 import litellm
 import litellm._logging
@@ -99,6 +99,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro
 from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
 from litellm.responses.utils import ResponseAPILoggingUtils
 from litellm.types.agents import LiteLLMSendMessageResponse
+from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage
 from litellm.types.llms.base import CachedTokensDetails
 from litellm.types.llms.openai import (
     HttpxBinaryResponseContent,
@@ -1023,6 +1024,8 @@ def get_usage_object(
         ),
     )
 
+    if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)):
+        return None if completion_response.usage is None else _decisions_usage(completion_response.usage)
     if usage_obj is None:
         return None
     if isinstance(usage_obj, Usage):
@@ -1053,11 +1056,37 @@ def get_usage_object(
         return None
 
 
+def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:
+    match usage:
+        case DecisionsUsage():
+            return PromptTokensDetailsWrapper(
+                cached_tokens=usage.cached_tokens, cache_write_tokens=usage.cache_write_tokens
+            )
+        case OpenAIDecisionUsage():
+            return PromptTokensDetailsWrapper(
+                cached_tokens=usage.input_tokens_details.cached_tokens,
+                cache_write_tokens=usage.input_tokens_details.cache_write_tokens,
+            )
+        case _:
+            assert_never(usage)
+
+
+def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
+    return Usage(
+        prompt_tokens=usage.input_tokens,
+        completion_tokens=usage.output_tokens,
+        total_tokens=usage.input_tokens + usage.output_tokens,
+        prompt_tokens_details=_decisions_prompt_tokens_details(usage),
+    )
+
+
 def _is_known_usage_objects(usage_obj):
     """Returns True if the usage obj is a known Usage type"""
     return (
         isinstance(usage_obj, litellm.Usage)
         or isinstance(usage_obj, ResponseAPIUsage)
+        or isinstance(usage_obj, DecisionsUsage)
+        or isinstance(usage_obj, OpenAIDecisionUsage)
         or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj)
     )
 
@@ -1466,7 +1495,9 @@ def completion_cost(
                             "usage",
                             litellm.Usage(**_usage_for_dump.model_dump()),
                         )
-                    if usage_obj is None:
+                    if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)):
+                        _usage = _decisions_usage(usage_obj).model_dump()
+                    elif usage_obj is None:
                         _usage = {}
                     elif isinstance(usage_obj, BaseModel):
                         _usage = cast(BaseModel, usage_obj).model_dump()
@@ -1957,7 +1988,9 @@ def response_cost_calculator(
     | LiteLLMRealtimeStreamLoggingObject
     | OpenAIModerationResponse
     | Response
-    | SearchResponse,
+    | SearchResponse
+    | DecisionsResponse
+    | OpenAIDecisionResponse,
     model: str,
     custom_llm_provider: str | None,
     call_type: Literal[
@@ -1979,6 +2012,10 @@ def response_cost_calculator(
         "arerank",
         "search",
         "asearch",
+        "decisions",
+        "adecisions",
+        "systemone",
+        "asystemone",
     ],
     optional_params: dict,
     cache_hit: bool | None = None,

litellm/decisions/__init__.py

@@ -0,0 +1,3 @@
+from litellm.decisions.main import adecisions, decisions
+
+__all__ = ["adecisions", "decisions"]

litellm/decisions/call.py

@@ -0,0 +1,347 @@
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field, replace
+from typing import Final, TypeAlias
+
+import httpx
+from pydantic import ConfigDict, TypeAdapter, ValidationError
+from typing_extensions import assert_never
+
+import litellm
+from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.decisions.transformation import (
+    BaseDecisionsConfig,
+    ir_to_systemone_response,
+    systemone_request_to_ir,
+)
+from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
+from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir
+from litellm.types.decisions import (
+    DecisionQuestion,
+    DecisionsIRRequest,
+    DecisionsIRResponse,
+    DecisionsJSON,
+    DecisionsRequestBody,
+    DecisionsResponse,
+    OpenAIDecisionInput,
+    OpenAIDecisionQuestion,
+    OpenAIDecisionRequestBody,
+    OpenAIDecisionResponse,
+    UnsupportedDecisionsRequest,
+)
+from litellm.types.utils import LlmProviders
+from litellm.utils import ProviderConfigManager
+
+SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
+OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
+DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody
+
+_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody)
+_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody)
+_SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(
+    str | None, config=ConfigDict(title="safety_identifier")
+)
+_HANDLER: Final = BaseLLMHTTPHandler()
+
+
+@dataclass(frozen=True, slots=True)
+class SystemOneRequestFields:
+    state: DecisionsJSON | None
+    questions: SystemOneQuestions | None
+
+    def validate(self, safety_identifier: str | None) -> DecisionsRequestBody:
+        return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": self.state, "questions": self.questions})
+
+
+@dataclass(frozen=True, slots=True)
+class OpenAIDecisionRequestFields:
+    input: OpenAIDecisionInput | None
+    questions: OpenAIDecisionQuestions | None
+
+    def validate(self, safety_identifier: str | None) -> OpenAIDecisionRequestBody:
+        return _OPENAI_REQUEST_ADAPTER.validate_python(
+            {"input": self.input, "questions": self.questions, "safety_identifier": safety_identifier}
+        )
+
+
+RequestFields: TypeAlias = SystemOneRequestFields | OpenAIDecisionRequestFields
+
+
+@dataclass(frozen=True, slots=True, repr=False)
+class DecisionsCall:
+    model: str
+    requested_model: str
+    custom_llm_provider: str
+    provider_config: BaseDecisionsConfig
+    request: DecisionsRequestFormat
+    ir_request: DecisionsIRRequest
+    body: Mapping[str, object] = field(repr=False)
+    api_base: str
+    api_key: str | None = field(repr=False)
+    logging_obj: LiteLLMLoggingObj | None
+    headers: Mapping[str, str]
+    timeout: float | httpx.Timeout | None
+    litellm_params: Mapping[str, object] = field(repr=False)
+
+
+def supported_decisions_providers() -> tuple[str, ...]:
+    return tuple(
+        provider.value
+        for provider in LlmProviders
+        if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None
+    )
+
+
+def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig:
+    provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None)
+    provider_config: Final = (
+        None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider)
+    )
+    if provider_config is None:
+        supported: Final = ", ".join(supported_decisions_providers())
+        raise litellm.BadRequestError(
+            message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}",
+            model=model,
+            llm_provider=custom_llm_provider,
+        )
+    return provider_config
+
+
+def _ir_request(request: DecisionsRequestFormat, safety_identifier: str | None) -> DecisionsIRRequest:
+    match request:
+        case DecisionsRequestBody():
+            return replace(systemone_request_to_ir(request), safety_identifier=safety_identifier)
+        case OpenAIDecisionRequestBody():
+            return openai_request_to_ir(request)
+        case _:
+            assert_never(request)
+
+
+def _drops_params(kwargs: Mapping[str, object]) -> bool:
+    return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True
+
+
+def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, object]) -> str | None:
+    if isinstance(safety_identifier, str) or not _drops_params(kwargs):
+        return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier)
+    return None
+
+
+@dataclass(frozen=True, slots=True)
+class _UnsupportedSafetyIdentifier:
+    pass
+
+
+def _provider_ir_request(
+    request: DecisionsRequestFormat,
+    *,
+    safety_identifier: str | None,
+    provider_config: BaseDecisionsConfig,
+    kwargs: Mapping[str, object],
+) -> DecisionsIRRequest | _UnsupportedSafetyIdentifier:
+    ir_request: Final = _ir_request(request, safety_identifier)
+    if ir_request.safety_identifier is None or provider_config.supports_safety_identifier:
+        return ir_request
+    if _drops_params(kwargs):
+        return replace(ir_request, safety_identifier=None)
+    return _UnsupportedSafetyIdentifier()
+
+
+def prepare_call(
+    *,
+    model: str,
+    fields: RequestFields,
+    safety_identifier: str | None,
+    api_key: str | None,
+    api_base: str | None,
+    timeout: float | httpx.Timeout | None,
+    custom_llm_provider: str | None,
+    extra_headers: Mapping[str, str] | None,
+    kwargs: Mapping[str, object],
+) -> DecisionsCall:
+    upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        api_base=api_base,
+        api_key=api_key,
+    )
+    provider_config: Final = _provider_config(upstream_model, provider)
+    if not upstream_model:
+        raise litellm.BadRequestError(
+            message="A model name is required for the Decisions API",
+            model=model,
+            llm_provider=provider,
+        )
+    try:
+        canonical_model: Final = provider_config.canonical_model(upstream_model)
+    except ValueError as error:
+        raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
+    try:
+        request_safety_identifier: Final = _request_safety_identifier(safety_identifier, kwargs)
+        request: Final = fields.validate(request_safety_identifier)
+    except ValidationError as error:
+        raise litellm.BadRequestError(
+            message=f"Invalid Decisions request: {error}",
+            model=model,
+            llm_provider=provider,
+        ) from error
+
+    resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base)
+    if resolved_api_base is None:
+        raise litellm.BadRequestError(
+            message=provider_config.missing_api_base_message(provider),
+            model=model,
+            llm_provider=provider,
+        )
+    resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key)
+    if resolved_api_key is None and provider_config.api_key_required:
+        raise litellm.AuthenticationError(
+            message=f"Missing API key for Decisions provider '{provider}'",
+            model=model,
+            llm_provider=provider,
+        )
+
+    ir_request: Final = _provider_ir_request(
+        request,
+        safety_identifier=request_safety_identifier,
+        provider_config=provider_config,
+        kwargs=kwargs,
+    )
+    if isinstance(ir_request, _UnsupportedSafetyIdentifier):
+        raise litellm.UnsupportedParamsError(
+            message=(
+                f"{provider} does not support parameters: ['safety_identifier'], for model={model}. "
+                "To drop these, set `litellm.drop_params=True` or for proxy:\n\n"
+                "`litellm_settings:\n drop_params: true`\n"
+            ),
+            model=model,
+            llm_provider=provider,
+        )
+    body: Final = provider_config.transform_decisions_request(
+        model=canonical_model, request=ir_request, custom_llm_provider=provider
+    )
+    if isinstance(body, UnsupportedDecisionsRequest):
+        raise litellm.BadRequestError(
+            message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}",
+            model=model,
+            llm_provider=provider,
+        )
+
+    logging_obj: Final = kwargs.get("litellm_logging_obj")
+    if isinstance(logging_obj, LiteLLMLoggingObj):
+        logging_obj.update_from_kwargs(
+            kwargs=dict(kwargs),
+            model=canonical_model,
+            litellm_params={
+                "litellm_call_id": kwargs.get("litellm_call_id"),
+                "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
+            },
+            custom_llm_provider=provider,
+        )
+    return DecisionsCall(
+        model=canonical_model,
+        requested_model=model,
+        custom_llm_provider=provider,
+        provider_config=provider_config,
+        request=request,
+        ir_request=ir_request,
+        body=body,
+        api_base=resolved_api_base,
+        api_key=resolved_api_key,
+        logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None,
+        headers=extra_headers or {},
+        timeout=timeout,
+        litellm_params=kwargs,
+    )
+
+
+def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
+    formatted: Final = _formatted_response(response, call)
+    provider_cost: Final = call.provider_config.provider_reported_cost(response)
+    formatted.set_hidden_params(
+        {
+            "model": f"{call.custom_llm_provider}/{call.model}",
+            "custom_llm_provider": call.custom_llm_provider,
+            "provider_response_model": f"{call.custom_llm_provider}/{call.model}",
+            **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
+        }
+    )
+    return formatted
+
+
+def _formatted_response(
+    response: DecisionsIRResponse, call: DecisionsCall
+) -> DecisionsResponse | OpenAIDecisionResponse:
+    match call.request:
+        case DecisionsRequestBody():
+            return ir_to_systemone_response(response, call.ir_request)
+        case OpenAIDecisionRequestBody():
+            return ir_to_openai_response(response, call.ir_request, call.requested_model)
+        case _:
+            assert_never(call.request)
+
+
+def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception:
+    if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
+        provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
+        return litellm.APIConnectionError(
+            message=f"{provider_label} - {error.message}",
+            llm_provider=call.custom_llm_provider,
+            model=f"{call.custom_llm_provider}/{call.model}",
+        )
+    return litellm.exception_type(
+        model=f"{call.custom_llm_provider}/{call.model}",
+        custom_llm_provider=call.custom_llm_provider,
+        original_exception=error,
+    )
+
+
+async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
+    try:
+        response: Final = await _HANDLER.adecisions(
+            model=call.model,
+            custom_llm_provider=call.custom_llm_provider,
+            logging_obj=call.logging_obj,
+            provider_config=call.provider_config,
+            request=call.ir_request,
+            body=call.body,
+            api_base=call.api_base,
+            api_key=call.api_key,
+            headers=call.headers,
+            timeout=call.timeout,
+            litellm_params=call.litellm_params,
+        )
+    except Exception as error:
+        raise _map_upstream_exception(error, call) from error
+    return _format_response(response, call)
+
+
+def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
+    try:
+        response: Final = _HANDLER.decisions(
+            model=call.model,
+            custom_llm_provider=call.custom_llm_provider,
+            logging_obj=call.logging_obj,
+            provider_config=call.provider_config,
+            request=call.ir_request,
+            body=call.body,
+            api_base=call.api_base,
+            api_key=call.api_key,
+            headers=call.headers,
+            timeout=call.timeout,
+            litellm_params=call.litellm_params,
+        )
+    except Exception as error:
+        raise _map_upstream_exception(error, call) from error
+    return _format_response(response, call)
+
+
+def reject_other_format(*, model: str, kwargs: Mapping[str, object], field_name: str, other_function: str) -> None:
+    if field_name not in kwargs:
+        return
+    raise litellm.BadRequestError(
+        message=f"Requests with '{field_name}' go through litellm.{other_function}",
+        model=model,
+        llm_provider="",
+    )

litellm/decisions/main.py

@@ -0,0 +1,131 @@
+import warnings
+from collections.abc import Mapping
+from typing import Final
+
+import httpx
+from pydantic import TypeAdapter
+
+from litellm.decisions.call import (
+    DecisionsCall,
+    OpenAIDecisionQuestions,
+    OpenAIDecisionRequestFields,
+    RequestFields,
+    SystemOneQuestions,
+    SystemOneRequestFields,
+    asend,
+    prepare_call,
+    send,
+)
+from litellm.types.decisions import DecisionsJSON, DecisionsResponse, OpenAIDecisionInput, OpenAIDecisionResponse
+from litellm.utils import client
+
+_STATE_ADAPTER: Final[TypeAdapter[DecisionsJSON | None]] = TypeAdapter(DecisionsJSON | None)
+_SYSTEMONE_QUESTIONS_ADAPTER: Final[TypeAdapter[SystemOneQuestions | None]] = TypeAdapter(SystemOneQuestions | None)
+_OPENAI_QUESTIONS_ADAPTER: Final[TypeAdapter[OpenAIDecisionQuestions | None]] = TypeAdapter(
+    OpenAIDecisionQuestions | None
+)
+
+
+def _request_fields(
+    decision_input: OpenAIDecisionInput | None,
+    questions: OpenAIDecisionQuestions | SystemOneQuestions | None,
+    kwargs: Mapping[str, object],
+) -> RequestFields:
+    if "state" not in kwargs:
+        return OpenAIDecisionRequestFields(
+            input=decision_input, questions=_OPENAI_QUESTIONS_ADAPTER.validate_python(questions)
+        )
+    warnings.warn(
+        "litellm.decisions(state=...) is deprecated, call litellm.systemone for the System One request format",
+        UserWarning,
+        stacklevel=5,
+    )
+    return SystemOneRequestFields(
+        state=_STATE_ADAPTER.validate_python(kwargs["state"]),
+        questions=_SYSTEMONE_QUESTIONS_ADAPTER.validate_python(questions),
+    )
+
+
+def _prepare_decisions_call(
+    *,
+    model: str,
+    decision_input: OpenAIDecisionInput | None,
+    questions: OpenAIDecisionQuestions | SystemOneQuestions | None,
+    safety_identifier: str | None,
+    api_key: str | None,
+    api_base: str | None,
+    timeout: float | httpx.Timeout | None,
+    custom_llm_provider: str | None,
+    extra_headers: Mapping[str, str] | None,
+    kwargs: Mapping[str, object],
+) -> DecisionsCall:
+    return prepare_call(
+        model=model,
+        fields=_request_fields(decision_input, questions, kwargs),
+        safety_identifier=safety_identifier,
+        api_key=api_key,
+        api_base=api_base,
+        timeout=timeout,
+        custom_llm_provider=custom_llm_provider,
+        extra_headers=extra_headers,
+        kwargs={key: value for key, value in kwargs.items() if key != "state"},
+    )
+
+
+@client
+async def adecisions(
+    model: str,
+    input: OpenAIDecisionInput | None = None,
+    questions: OpenAIDecisionQuestions | SystemOneQuestions | None = None,
+    safety_identifier: str | None = None,
+    api_key: str | None = None,
+    api_base: str | None = None,
+    timeout: float | httpx.Timeout | None = None,
+    custom_llm_provider: str | None = None,
+    extra_headers: Mapping[str, str] | None = None,
+    **kwargs: object,
+) -> OpenAIDecisionResponse | DecisionsResponse:
+    call: Final = _prepare_decisions_call(
+        model=model,
+        decision_input=input,
+        questions=questions,
+        safety_identifier=safety_identifier,
+        api_key=api_key,
+        api_base=api_base,
+        timeout=timeout,
+        custom_llm_provider=custom_llm_provider,
+        extra_headers=extra_headers,
+        kwargs=kwargs,
+    )
+    return await asend(call)
+
+
+@client
+def decisions(
+    model: str,
+    input: OpenAIDecisionInput | None = None,
+    questions: OpenAIDecisionQuestions | SystemOneQuestions | None = None,
+    safety_identifier: str | None = None,
+    api_key: str | None = None,
+    api_base: str | None = None,
+    timeout: float | httpx.Timeout | None = None,
+    custom_llm_provider: str | None = None,
+    extra_headers: Mapping[str, str] | None = None,
+    **kwargs: object,
+) -> OpenAIDecisionResponse | DecisionsResponse:
+    call: Final = _prepare_decisions_call(
+        model=model,
+        decision_input=input,
+        questions=questions,
+        safety_identifier=safety_identifier,
+        api_key=api_key,
+        api_base=api_base,
+        timeout=timeout,
+        custom_llm_provider=custom_llm_provider,
+        extra_headers=extra_headers,
+        kwargs=kwargs,
+    )
+    return send(call)
+
+
+__all__ = ["adecisions", "decisions"]

litellm/integrations/gitlab/gitlab_prompt_manager.py

@@ -215,7 +215,7 @@ class GitLabTemplateManager:
                 elif value.replace(".", "").isdigit():
                     try:
                         result[key] = float(value)
-                    except Exception:
+                    except ValueError:
                         result[key] = value
                 else:
                     result[key] = value.strip("\"'")

litellm/integrations/otel/logger.py

@@ -915,7 +915,8 @@ def _excluded_db_systems(logger: "OpenTelemetryV2") -> frozenset[str]:
     logger got published: with ``callbacks: [langfuse_otel, otel]`` the ``otel``
     callback folds into the preset, whose config is env-only.
     """
-    configured: Final = litellm.callback_settings.get("otel", {}).get("excluded_services")
+    otel_settings: Final = (litellm.callback_settings or {}).get("otel")
+    configured: Final = otel_settings.get("excluded_services") if isinstance(otel_settings, dict) else None
     if configured is None:
         return logger.config.excluded_services
     return excluded_db_systems_from(configured)

litellm/integrations/otel/model/config.py

@@ -5,7 +5,8 @@ from functools import lru_cache
 from typing import Annotated, Any, Final
 
 from pydantic import AliasChoices, BaseModel, Field, TypeAdapter, ValidationError, field_validator, model_validator
-from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
+from pydantic.fields import FieldInfo
+from pydantic_settings import BaseSettings, NoDecode, PydanticBaseSettingsSource, SettingsConfigDict
 
 from litellm._logging import verbose_logger
 from litellm.integrations.otel.model.baggage import (
@@ -121,9 +122,37 @@ class ExporterSpec(BaseModel):
     )
 
 
+class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
+    def __init__(self, settings_cls: type[BaseSettings], env_settings: PydanticBaseSettingsSource) -> None:
+        super().__init__(settings_cls)
+        self._env_settings: Final = env_settings
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[object, str, bool]:
+        return self._env_settings.get_field_value(field, field_name)
+
+    def __call__(self) -> dict[str, object]:
+        return {key: value for key, value in self._env_settings().items() if key != "excluded_services"}
+
+
 class OpenTelemetryV2Config(BaseSettings):
     model_config = SettingsConfigDict(populate_by_name=True, extra="ignore")
 
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        return (
+            init_settings,
+            _EnvWithoutBareExcludedServices(settings_cls, env_settings),
+            dotenv_settings,
+            file_secret_settings,
+        )
+
     # ----- single-destination shorthand, read from standard OTEL_* envs ----- #
     exporter: str = Field(
         default="console",
@@ -178,7 +207,7 @@ class OpenTelemetryV2Config(BaseSettings):
     )
     excluded_services: Annotated[frozenset[str], NoDecode] = Field(
         default_factory=frozenset,
-        validation_alias=AliasChoices("excluded_services", "LITELLM_OTEL_EXCLUDED_SERVICES"),
+        validation_alias=AliasChoices("LITELLM_OTEL_EXCLUDED_SERVICES"),
         description=(
             "Datastore services whose spans are withheld from key/team ``callback_vars`` "
             "OTel destinations (the operator's own exporters still receive them). Accepted "

litellm/litellm_core_utils/core_helpers.py

@@ -3,9 +3,9 @@
 import copy
 import logging
 import re
-from collections.abc import Collection, Iterable, Mapping
+from collections.abc import Collection, Iterable, Iterator, Mapping
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, runtime_checkable
 
 import httpx
 from pydantic import TypeAdapter, ValidationError
@@ -51,6 +51,28 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non
 _DROP_PARAMS_BOOL: Final = TypeAdapter(bool)
 
 
+def as_str_mapping(value: object) -> Mapping[str, object] | None:
+    return value if isinstance(value, Mapping) else None
+
+
+@runtime_checkable
+class _SupportsIter(Protocol):
+    def __iter__(self) -> Iterator[object]: ...
+
+
+@runtime_checkable
+class _SupportsGetItem(Protocol):
+    def __getitem__(self, index: int, /) -> object: ...
+
+
+def iter_items(value: object) -> Iterable[object]:
+    if isinstance(value, _SupportsIter):
+        return value
+    if isinstance(value, _SupportsGetItem):
+        return iter(value)
+    raise TypeError(f"'{type(value).__name__}' object is not iterable")
+
+
 def normalize_drop_params(value: object) -> bool | None:
     if value is None or isinstance(value, bool):
         return value

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -2146,7 +2146,10 @@ def _map_openrouter_exception(
     exception_provider: str,
     extra_information: str,
 ) -> None:
-    if hasattr(original_exception, "status_code"):
+    received_status: Final = hasattr(original_exception, "status_code") and not getattr(
+        original_exception, "status_code_is_synthesized", False
+    )
+    if received_status:
         if original_exception.status_code == 400:
             raise BadRequestError(
                 message=f"{exception_provider} - {error_str}",

litellm/litellm_core_utils/health_check_helpers.py

@@ -3,7 +3,7 @@ Helper functions for health check calls.
 """
 
 import base64
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Mapping
 from typing import TYPE_CHECKING, Final, Literal
 
 from litellm.llms.base_llm.ocr.transformation import DocumentType
@@ -30,6 +30,21 @@ def get_image_file_for_health_check() -> bytes:
     return base64.b64decode(TEST_IMAGE_BASE64)
 
 
+def _decisions_health_check_questions(model: str, custom_llm_provider: str) -> Mapping[str, Mapping[str, object]]:
+    import litellm
+    from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+    from litellm.utils import ProviderConfigManager
+
+    provider: Final = next((member for member in litellm.LlmProviders if member.value == custom_llm_provider), None)
+    config: Final = (
+        None
+        if provider is None
+        else ProviderConfigManager.get_provider_decisions_config(model=model, provider=provider)
+    )
+    questions: Final = BaseDecisionsConfig.health_check_questions if config is None else config.health_check_questions
+    return {name: dict(question) for name, question in questions.items()}
+
+
 def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType:
     native: Final = NATIVE_OCR_HEALTH_CHECK_DOCUMENT.load()
     if native is None:
@@ -168,6 +183,7 @@ class HealthCheckHelpers:
             "batch",
             "responses",
             "ocr",
+            "evaluation",
         ],
         Callable,
     ]:
@@ -190,7 +206,7 @@ class HealthCheckHelpers:
         from litellm.litellm_core_utils.audio_utils.utils import (
             get_audio_file_for_health_check,
         )
-        from litellm.litellm_core_utils.health_check_utils import _filter_model_params
+        from litellm.litellm_core_utils.health_check_utils import DECISIONS_CALL_PARAMS, _filter_model_params
         from litellm.realtime_api.main import _realtime_health_check
 
         return {
@@ -257,4 +273,15 @@ class HealthCheckHelpers:
                 **_filter_model_params(model_params=model_params),
                 document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider),
             ),
+            "evaluation": lambda: litellm.asystemone(
+                **DECISIONS_CALL_PARAMS.validate_python(
+                    {
+                        "state": prompt or "health check",
+                        "questions": _decisions_health_check_questions(
+                            model=model, custom_llm_provider=custom_llm_provider
+                        ),
+                        **_filter_model_params(model_params=model_params),
+                    }
+                )
+            ),
         }

litellm/litellm_core_utils/health_check_utils.py

@@ -4,6 +4,12 @@ Utils used for litellm.ahealth_check()
 
 from typing import Final
 
+from pydantic import TypeAdapter
+
+from litellm.types.decisions import DecisionsCallParams
+
+DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams)
+
 
 def _filter_model_params(model_params: dict) -> dict:
     """Remove 'messages' param from model params."""

litellm/litellm_core_utils/litellm_logging.py

@@ -118,6 +118,7 @@ from litellm.llms.base_llm.search.transformation import SearchResponse
 from litellm.responses.utils import ResponseAPILoggingUtils
 from litellm.types.agents import LiteLLMSendMessageResponse
 from litellm.types.containers.main import ContainerObject
+from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse
 from litellm.types.integrations.s3_v2 import S3PartitionGranularity
 from litellm.types.interactions import (
     InteractionsAPIResponse,
@@ -679,6 +680,9 @@ class Logging(LiteLLMLoggingBaseClass):
         self.truncated_messages_for_logging: str | list | dict | None = None  # mutable-ok: logged messages shape
         ## TIME TO FIRST TOKEN LOGGING ##
         self.completion_start_time: datetime.datetime | None = None
+        # The model the proxy shows the client on streamed chunks. The logged streamed response carries it
+        # once that response is priced, the same way a non-streamed response is logged
+        self.client_facing_stream_model: str | None = None
         self.zero_cost_warned: bool = False
         self._llm_caching_handler: LLMCachingHandler | None = None
 
@@ -717,6 +721,7 @@ class Logging(LiteLLMLoggingBaseClass):
         # enqueue closure here instead of firing it immediately.
         self._defer_async_logging: bool = False
         self._enqueue_deferred_logging: Callable[[], None] | None = None
+        self._async_success_scheduled: bool = False
         self._on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
@@ -1812,6 +1817,7 @@ class Logging(LiteLLMLoggingBaseClass):
             LiteLLMRealtimeStreamLoggingObject,
             OpenAIModerationResponse,
             "SearchResponse",
+            DecisionsResponse,
             dict,
             list,
         ],
@@ -2254,6 +2260,15 @@ class Logging(LiteLLMLoggingBaseClass):
         except Exception:
             return True
 
+    def claim_async_success_log(self) -> bool:
+        """One async success log per request. The innermost @client wrapper always exits first,
+        since the outer one is awaiting it, so it claims the log here and the outer wrapper gets
+        False and schedules nothing."""
+        if self._async_success_scheduled:
+            return False
+        self._async_success_scheduled = True
+        return True
+
     def has_run_logging(
         self,
         event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"],
@@ -2482,6 +2497,15 @@ class Logging(LiteLLMLoggingBaseClass):
             setattr(result, "usage", transformed_usage)
         return result
 
+    def _with_client_facing_stream_model(
+        self,
+        response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
+    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
+        model: Final = self.client_facing_stream_model
+        if model is None or getattr(response, "model", None) in (None, model):
+            return response
+        return response.model_copy(update={"model": model})
+
     def _success_handler_helper_fn(
         self,
         result=None,
@@ -2588,6 +2612,8 @@ class Logging(LiteLLMLoggingBaseClass):
             or isinstance(logging_result, OpenAIModerationResponse)
             or isinstance(logging_result, OCRResponse)  # OCR
             or isinstance(logging_result, SearchResponse)  # Search API
+            or isinstance(logging_result, DecisionsResponse)
+            or isinstance(logging_result, OpenAIDecisionResponse)
             or (
                 isinstance(logging_result, InteractionsAPIResponse)
                 and logging_result.usage is not None
@@ -2797,9 +2823,11 @@ class Logging(LiteLLMLoggingBaseClass):
                     result=complete_streaming_response
                 )
                 self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
+                logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
+                self.model_call_details["complete_streaming_response"] = logged_streaming_response
                 ## STANDARDIZED LOGGING PAYLOAD
                 self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
-                    complete_streaming_response, start_time, end_time
+                    logged_streaming_response, start_time, end_time
                 )
                 standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
                     "standard_logging_object"
@@ -3338,10 +3366,13 @@ class Logging(LiteLLMLoggingBaseClass):
 
             await self._prepare_baseline_cache_estimate(complete_streaming_response)
 
+            logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
+            self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
+
             ## STANDARDIZED LOGGING PAYLOAD
             try:
                 self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
-                    complete_streaming_response, start_time, end_time
+                    logged_streaming_response, start_time, end_time
                 )
             except Exception:  # noqa: BLE001  # payload build must never block later callbacks (slot release)
                 verbose_logger.exception(

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1993,10 +1993,8 @@ def is_unsignable_thinking_block(block: object) -> bool:
     Anthropic verifies the signature cryptographically, so a block with a null,
     empty, or missing signature (e.g. from an open-source reasoning model) is
     rejected with a 400, and so is a block whose signature or data carries
-    another provider's encrypted reasoning. It also rejects a `thinking` block
+    another provider's encrypted reasoning. A signed block is kept whatever its
-    whose text is empty or whitespace-only ("each thinking block must contain
+    text, since ``display: omitted`` returns signed blocks with empty ``thinking``.
-    thinking"), regardless of signature, e.g. when a `thinking_blocks` history
-    item from a non-Anthropic reasoning provider is replayed through this path.
     `redacted_thinking` blocks carry no signature and are always kept.
     """
     if is_encrypted_reasoning_block(block):
@@ -2007,10 +2005,7 @@ def is_unsignable_thinking_block(block: object) -> bool:
     if mapping.get("type") != "thinking":
         return False
     signature: Final = mapping.get("signature")
-    if not (isinstance(signature, str) and len(signature) > 0):
+    return not (isinstance(signature, str) and len(signature) > 0)
-        return True
-    thinking_text: Final = mapping.get("thinking")
-    return not (isinstance(thinking_text, str) and len(thinking_text.strip()) > 0)
 
 
 def strip_encrypted_reasoning_from_messages(

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -2684,13 +2684,8 @@ def anthropic_messages_pt(
                         if not isinstance(m, dict):
                             continue
                         # handle thinking blocks
-                        thinking_block = cast(str, m.get("thinking", ""))
                         text_block = cast(str, m.get("text", ""))
-                        if (
+                        if m.get("type", "") == "thinking" and not is_unsignable_thinking_block(m):
-                            m.get("type", "") == "thinking"
-                            and len(thinking_block) > 0
-                            and not is_unsignable_thinking_block(m)
-                        ):  # don't pass empty text blocks. anthropic api raises errors.
                             anthropic_message: ChatCompletionThinkingBlock | AnthropicMessagesTextParam = cast(
                                 ChatCompletionThinkingBlock, m
                             )
@@ -5671,3 +5666,25 @@ def resolve_structured_messages(
         if structured:
             return [msg if isinstance(msg, dict) else msg.model_dump() for msg in structured]
     return None
+
+
+def is_flat_responses_tool(tool: Mapping[str, object]) -> bool:
+    tool_type: Final = tool.get("type")
+    return tool_type in ("function", "custom") and tool_type not in tool and "input_schema" not in tool
+
+
+def chat_completion_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
+    from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
+    from litellm.responses.litellm_completion_transformation.transformation import (
+        LiteLLMCompletionResponsesConfig,
+    )
+
+    if is_flat_responses_tool(tool):
+        chat_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
+            [dict(tool)]  # pyright: ignore[reportArgumentType]  # a JSON dict in a flat Responses tool shape
+        )
+        return tuple(chat_tools)
+    anthropic_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(
+        [dict(tool)]  # pyright: ignore[reportArgumentType]  # a JSON dict in an Anthropic or chat tool shape
+    )
+    return tuple(anthropic_tools)

litellm/litellm_core_utils/prompt_templates/image_handling.py

@@ -4,8 +4,9 @@ Helper functions to handle images passed in messages
 
 import asyncio
 import base64
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Iterable, Mapping
 from dataclasses import dataclass
+from itertools import chain
 from types import MappingProxyType
 from typing import Final
 
@@ -15,6 +16,7 @@ import litellm
 from litellm import verbose_logger
 from litellm.caching.caching import InMemoryCache
 from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB
+from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content
 from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
 from litellm.types.llms.openai import AllMessageValues
 
@@ -55,23 +57,16 @@ def _process_image_response(response: Response, url: str) -> str:
 
     base64_image: Final = base64.b64encode(image_bytes).decode("utf-8")
 
-    image_type: Final = response.headers.get("Content-Type")
+    try:
-    if image_type is None:
+        img_type: Final = infer_content_type_from_url_and_content(
-        img_type = url.split(".")[-1].lower()
+            url=url,
-        _img_type: Final = {
+            content=bytes(image_bytes),
-            "jpg": "image/jpeg",
+            current_content_type=response.headers.get("Content-Type"),
-            "jpeg": "image/jpeg",
+        )
-            "png": "image/png",
+    except ValueError as e:
-            "gif": "image/gif",
+        raise litellm.ImageFetchError(
-            "webp": "image/webp",
+            f"Error: Unable to determine image content type from the server's headers, the URL, or the image bytes. url={url}"
-        }.get(img_type)
+        ) from e
-        if _img_type is None:
-            raise Exception(
-                f"Error: Unsupported image format. Format={_img_type}. Supported types = ['image/jpeg', 'image/png', 'image/gif', 'image/webp']"
-            )
-        img_type = _img_type
-    else:
-        img_type = image_type
 
     result: Final = f"data:{img_type};base64,{base64_image}"
     in_memory_cache.set_cache(url, result)
@@ -308,18 +303,30 @@ async def _fetch_data_urls(remote_urls: tuple[str, ...]) -> tuple[str, ...]:
         raise
 
 
+def _remote_urls_to_inline(
+    messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool]
+) -> tuple[str, ...]:
+    parts: Final = chain.from_iterable(_content_parts(message) for message in messages)
+    remotes: Final = (remote for part in parts if (remote := _parse_remote_part(part)) is not None)
+    return tuple(dict.fromkeys(remote.url for remote in remotes if should_inline(_remote_media(remote))))
+
+
+def inline_remote_media(
+    messages: list[AllMessageValues],  # mutable-ok: every transform_request takes list[AllMessageValues]
+    should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url,
+) -> list[AllMessageValues]:  # mutable-ok: every transform_request takes list[AllMessageValues]
+    remote_urls: Final = _remote_urls_to_inline(messages, should_inline)
+    if not remote_urls:
+        return messages
+    data_urls: Final = MappingProxyType({url: convert_url_to_base64(url) for url in remote_urls})
+    return [_inline_message(message, data_urls, should_inline) for message in messages]
+
+
 async def async_inline_remote_media(
     messages: list[AllMessageValues],  # mutable-ok: every transform_request takes list[AllMessageValues]
     should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url,
 ) -> list[AllMessageValues]:  # mutable-ok: every transform_request takes list[AllMessageValues]
-    remote_urls: Final = tuple(
+    remote_urls: Final = _remote_urls_to_inline(messages, should_inline)
-        dict.fromkeys(
-            remote.url
-            for message in messages
-            for part in _content_parts(message)
-            if (remote := _parse_remote_part(part)) is not None and should_inline(_remote_media(remote))
-        )
-    )
     if not remote_urls:
         return messages
     data_urls: Final = await _fetch_data_urls(remote_urls)

litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py

@@ -257,8 +257,7 @@ def _assistant_part_renders(part: object) -> bool:
     """A text part always renders: the converter pads empty text with a placeholder."""
     part_type: Final = message_field(part, "type")
     if part_type == "thinking":
-        thinking: Final = message_field(part, "thinking")
+        return _thinking_block_renders(part)
-        return isinstance(thinking, str) and bool(thinking) and _thinking_block_renders(part)
     return part_type in _RENDERED_ASSISTANT_PART_TYPES or (
         isinstance(part_type, str) and part_type.endswith("_tool_result")
     )

litellm/llms/anthropic/chat/transformation.py

@@ -327,6 +327,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
     def _resolved_provider(self) -> str:
         return self.custom_llm_provider or "anthropic"
 
+    @property
+    def _carries_thinking_block_binding(self) -> bool:
+        return type(self) is AnthropicConfig
+
     @classmethod
     def get_config(cls, *, model: str | None = None):
         config: Final = super().get_config()
@@ -1588,7 +1592,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 else:
                     optional_params["thinking"] = value
                     AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
-                        model=model, optional_params=optional_params, custom_llm_provider=self._resolved_provider
+                        model=model,
+                        optional_params=optional_params,
+                        custom_llm_provider=self._resolved_provider,
+                        carry_block_binding=self._carries_thinking_block_binding,
                     )
             elif param == "reasoning_effort":
                 # Accept both string ("low") and dict ({"effort": "low",

litellm/llms/anthropic/common_utils.py

@@ -78,7 +78,7 @@ _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
 _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
 _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
 _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
-_CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
+_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
 _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
 
 
@@ -127,13 +127,18 @@ def is_claude_code_user_agent(user_agent: str) -> bool:
     return user_agent.startswith(_CLAUDE_CODE_USER_AGENT_PREFIXES)
 
 
-def _validated_claude_code_mapping(value: object) -> dict[object, object] | None:
+def validated_object_mapping(value: object) -> dict[object, object] | None:
     try:
-        return _CLAUDE_CODE_OBJECT_MAPPING_ADAPTER.validate_python(value)
+        return _OBJECT_MAPPING_ADAPTER.validate_python(value)
     except ValidationError:
         return None
 
 
+def thinking_block_binding(thinking: Mapping[object, object] | None) -> dict[str, object]:
+    block_binding: Final = thinking.get("block_binding") if thinking is not None else None
+    return {} if block_binding is None else {"block_binding": block_binding}
+
+
 def _validated_claude_code_list(value: object) -> list[object] | None:
     try:
         return _CLAUDE_CODE_OBJECT_LIST_ADAPTER.validate_python(value)
@@ -166,7 +171,7 @@ def _claude_code_billing_texts(system: object) -> tuple[str, ...] | None:
     blocks: Final = _validated_claude_code_list(system)
     if blocks is None:
         return None
-    block_mappings: Final = tuple(_validated_claude_code_mapping(block) for block in blocks)
+    block_mappings: Final = tuple(validated_object_mapping(block) for block in blocks)
     if any(block is None for block in block_mappings):
         return None
     text_values: Final = tuple(
@@ -199,7 +204,7 @@ def is_claude_code_one_shot_subagent_request(
     tools: object,
     user_agent: str | None,
 ) -> bool:
-    only_message: Final = _validated_claude_code_mapping(messages[0]) if len(messages) == 1 else None
+    only_message: Final = validated_object_mapping(messages[0]) if len(messages) == 1 else None
     return (
         user_agent is not None
         and is_claude_code_user_agent(user_agent)
@@ -748,6 +753,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         model: str,
         optional_params: MutableMapping[str, object],  # mutable-ok: in-place out-param like the sibling helpers
         custom_llm_provider: str,
+        carry_block_binding: bool = False,
     ) -> None:
         """Translate legacy ``thinking.type=enabled`` to adaptive for the
         adaptive-thinking models that reject it (4.7+ and the 5 families).
@@ -760,21 +766,21 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             return
         if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
             return
-        thinking: Final = optional_params.get("thinking")
+        thinking: Final = validated_object_mapping(optional_params.get("thinking"))
-        if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
+        if thinking is None or thinking.get("type") != "enabled":
             return
 
+        budget_tokens: Final = thinking.get("budget_tokens") or 0
         effort: Final = AnthropicModelInfo._legacy_budget_to_effort(
             model=model,
-            budget_tokens=int(thinking.get("budget_tokens") or 0),
+            budget_tokens=int(budget_tokens) if isinstance(budget_tokens, (int, float, str)) else 0,
             custom_llm_provider=custom_llm_provider,
         )
         existing_output_config: Final = optional_params.get("output_config")
         display: Final = thinking.get("display")
-        if display in ("summarized", "omitted"):
+        supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {}
-            optional_params["thinking"] = {"type": "adaptive", "display": display}
+        carried_block_binding: Final = thinking_block_binding(thinking) if carry_block_binding else {}
-        else:
+        optional_params["thinking"] = {**carried_block_binding, "type": "adaptive", **supported_display}
-            optional_params["thinking"] = {"type": "adaptive"}
         optional_params["output_config"] = {
             "effort": effort,
             **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})),
@@ -1320,6 +1326,20 @@ def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool:
     return "must contain thinking" in lower
 
 
+def is_anthropic_thinking_binding_mismatch_error(error_text: str) -> bool:
+    """messages.N.content.M: Invalid `signature` in `thinking` block. The block is bound to a different conversation."""
+    return "bound to a different conversation" in error_text.lower()
+
+
+def anthropic_request_errors_on_thinking_binding_mismatch(request_body: object) -> bool:
+    """``thinking.block_binding.prefix_mismatch_behavior: error`` is the caller asking
+    for the 400, so a retry that strips thinking must not override it."""
+    body: Final = validated_object_mapping(request_body)
+    thinking: Final = validated_object_mapping(body.get("thinking")) if body is not None else None
+    binding: Final = validated_object_mapping(thinking.get("block_binding")) if thinking is not None else None
+    return binding is not None and binding.get("prefix_mismatch_behavior") == "error"
+
+
 def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]:
     """
     Return a new message list with thinking / redacted_thinking content blocks removed
@@ -1390,7 +1410,7 @@ def strip_empty_content_blocks_from_anthropic_messages(
 ) -> list[Any]:
     """
     Return a new message list with empty or whitespace-only ``{"type": "text"}``
-    and ``{"type": "thinking"}`` content blocks removed.
+    content blocks and empty *unsigned* ``{"type": "thinking"}`` blocks removed.
 
     Anthropic's API rejects requests containing such blocks with
     ``"messages: text content blocks must be non-empty"`` and
@@ -1406,8 +1426,13 @@ def strip_empty_content_blocks_from_anthropic_messages(
     on the unified ``/v1/messages`` path.  ``/v1/chat/completions`` already
     handles this in ``anthropic_messages_pt``; this helper provides the
     equivalent guarantee for the native Anthropic Messages path.
-    ``redacted_thinking`` blocks are never touched: they carry opaque
+
-    ``data`` instead of thinking text.
+    A thinking block with empty text but a non-empty ``signature`` is kept:
+    it is the default response shape of Claude Opus 4.8 / Fable 5 / Fable 5.1
+    (``thinking.display`` omitted), the signature is the model's preserved
+    reasoning, and Anthropic accepts it on input.  ``redacted_thinking``
+    blocks are never touched: they carry opaque ``data`` instead of thinking
+    text.
 
     Messages whose content is a list and becomes empty after stripping are
     omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`.
@@ -1420,7 +1445,7 @@ def strip_empty_content_blocks_from_anthropic_messages(
             out.append(m)
             continue
         content = m["content"]
-        filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_thinking_block(b)]
+        filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_unsigned_thinking_block(b)]
         if len(filtered) == len(content):
             out.append(m)
         elif filtered:
@@ -1438,11 +1463,11 @@ def _is_empty_text_block(block: object) -> bool:
 def is_empty_thinking_block(block: object) -> bool:
     """
     True for a ``{"type": "thinking"}`` content block whose thinking text is
-    missing, not a string, or empty/whitespace-only after ``.strip()``.
+    missing, not a string, or empty/whitespace-only after ``.strip()``,
-    Anthropic rejects such blocks with ``"each thinking block must contain
+    whatever its signature.  This is a shape check only; whether such a block
-    thinking"`` (whitespace-only included, verified live), regardless of any
+    may be sent to Anthropic depends on the signature, see
-    signature they carry.  ``redacted_thinking`` blocks are a different type
+    :func:`is_empty_unsigned_thinking_block`.  ``redacted_thinking`` blocks are
-    and always return False.
+    a different type and always return False.
     """
     if not isinstance(block, dict) or block.get("type") != "thinking":
         return False
@@ -1454,15 +1479,16 @@ def is_empty_unsigned_thinking_block(block: object) -> bool:
     """
     True for an empty ``{"type": "thinking"}`` block carrying no signature.
 
-    The emit-side predicate: response paths drop a thinking block only when it
+    Used on both the emit and the request side: a thinking block is dropped
-    holds nothing the client could need.  A signature-only block is a real
+    only when it holds nothing the provider or the client could use.  A
-    provider response (Bedrock Converse under adaptive thinking emits a
+    signature-only block is a real provider response (Claude Opus 4.8 /
-    reasoning block with empty text and only a signature) and the client needs
+    Fable 5 / Fable 5.1 return it by default, and Bedrock Converse under
-    the signature to replay reasoning across tool-use turns, so it must be
+    adaptive thinking emits a reasoning block with empty text and only a
-    emitted.  Request paths keep using :func:`is_empty_thinking_block`:
+    signature); the client needs the signature to replay reasoning across
-    Anthropic rejects empty thinking blocks in request history regardless of
+    tool-use turns, and Anthropic accepts the replayed block and binds it to
-    signature, and the inbound strip self-heals a replayed signature-only
+    its prefix.  Only an empty block with no signature (a turn a
-    block.
+    non-Anthropic reasoning model served through the bridge) is rejected
+    with ``"each thinking block must contain thinking"`` and must go.
     """
     if not isinstance(block, dict) or not is_empty_thinking_block(block):
         return False

litellm/llms/anthropic/count_tokens/token_counter.py

@@ -5,10 +5,13 @@ Anthropic Token Counter implementation using the CountTokens API.
 import os
 from typing import Any, Final
 
+from pydantic import JsonValue
+
+import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
-from litellm.types.utils import LlmProviders, TokenCountResponse
+from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
 
 # Global handler instance - reuse across all token counting requests
 anthropic_count_tokens_handler: Final = AnthropicCountTokensHandler()
@@ -30,8 +33,10 @@ class AnthropicTokenCounter(BaseTokenCounter):
         contents: list[dict[str, Any]] | None,
         deployment: dict[str, Any] | None = None,
         request_model: str = "",
-        tools: list[dict[str, Any]] | None = None,
+        tools: list[dict[str, JsonValue]] | None = None,
-        system: Any | None = None,
+        system: JsonValue = None,
+        messages_format: TokenCountMessagesFormat = "anthropic",
+        output_format: JsonValue = None,
     ) -> TokenCountResponse | None:
         """
         Count tokens using Anthropic's CountTokens API.
@@ -54,14 +59,22 @@ class AnthropicTokenCounter(BaseTokenCounter):
         deployment = deployment or {}
         litellm_params: Final = deployment.get("litellm_params", {})
 
-        # Get Anthropic API key from deployment config or environment
+        api_key: Final = (
-        api_key = litellm_params.get("api_key")
+            litellm_params.get("api_key") or litellm.anthropic_key or litellm.api_key or os.getenv("ANTHROPIC_API_KEY")
+        )
         if not api_key:
-            api_key = os.getenv("ANTHROPIC_API_KEY")
+            return TokenCountResponse(
-
+                total_tokens=0,
-        if not api_key:
+                request_model=request_model,
-            verbose_logger.warning("No Anthropic API key found for token counting")
+                model_used=model_to_use,
-            return None
+                tokenizer_type="anthropic_api",
+                error=True,
+                error_message=(
+                    f"No Anthropic API key is set for model {request_model or model_to_use}, so tokens can't be "
+                    "counted. Set api_key on the deployment or ANTHROPIC_API_KEY in the proxy environment."
+                ),
+                status_code=401,
+            )
 
         try:
             result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request(

litellm/llms/anthropic/pass_through/adapters/transformation.py

@@ -460,7 +460,7 @@ class LiteLLMAnthropicMessagesAdapter:
         *,
         custom_llm_provider: str | None = None,
         preserve_midturn_system: bool = False,
-    ) -> list:
+    ) -> list[AllMessageValues]:
         new_messages: Final[list[AllMessageValues]] = []
         replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
         leading_count: Final = next(

litellm/llms/anthropic/pass_through/convertible_blocks.py

@@ -0,0 +1,106 @@
+from collections.abc import Iterator, Mapping, Sequence
+from itertools import chain
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+from typing_extensions import assert_never
+
+from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
+from litellm.types.utils import TokenCountMessagesFormat
+
+_CONVERTIBLE_BLOCK_TYPES: Final = MappingProxyType(
+    {
+        "user": frozenset({"text", "image", "document", "tool_result"}),
+        "assistant": frozenset({"text", "tool_use", "thinking", "redacted_thinking"}),
+    }
+)
+_SOURCED_BLOCK_TYPES: Final = frozenset({"image", "document"})
+_TOOL_RESULT_BLOCK_TYPES: Final = frozenset({"text", "image", "document"})
+
+
+_JSON_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
+
+
+def _json_messages(messages: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, JsonValue], ...]:
+    try:
+        return _JSON_MESSAGES.validate_python(messages)
+    except ValidationError:
+        return ()
+
+
+def _source_label(block_type: str, source: JsonValue) -> str | None:
+    if not isinstance(source, dict):
+        return f"{block_type} without a source"
+    match source.get("type"):
+        case "base64" if source.get("data"):
+            return None
+        case "url" if source.get("url"):
+            return None
+        case "base64" | "url" as source_type:
+            return f"{block_type} with an empty {source_type} source"
+        case str() as source_type:
+            return f"{block_type} with a {source_type} source"
+        case _:
+            return f"{block_type} with an untyped source"
+
+
+def _block_label(block: JsonValue, convertible: frozenset[str]) -> str | None:
+    if not isinstance(block, dict):
+        return None
+    block_type: Final = block.get("type")
+    if not isinstance(block_type, str):
+        return "a block without a type"
+    if block_type not in convertible:
+        return block_type
+    return _source_label(block_type, block.get("source")) if block_type in _SOURCED_BLOCK_TYPES else None
+
+
+def _tool_result_labels(block: JsonValue) -> Iterator[str]:
+    if not isinstance(block, dict) or block.get("type") != "tool_result":
+        return iter(())
+    inner: Final = block.get("content")
+    if not isinstance(inner, list):
+        return iter(())
+    return (
+        f"{label} inside a tool_result"
+        for item in inner
+        if (label := _block_label(item, _TOOL_RESULT_BLOCK_TYPES)) is not None
+    )
+
+
+def _content_labels(block: JsonValue, convertible: frozenset[str]) -> Iterator[str]:
+    label: Final = _block_label(block, convertible)
+    return chain(() if label is None else (label,), _tool_result_labels(block))
+
+
+def _message_labels(message: Mapping[str, JsonValue]) -> Iterator[str]:
+    role: Final = message.get("role")
+    convertible: Final = _CONVERTIBLE_BLOCK_TYPES.get(role) if isinstance(role, str) else None
+    content: Final = message.get("content")
+    if convertible is None or not isinstance(content, list):
+        return iter(())
+    return chain.from_iterable(_content_labels(block, convertible) for block in content)
+
+
+def unconvertible_anthropic_blocks(messages: Sequence[Mapping[str, object]]) -> tuple[str, ...]:
+    normalized: Final = _json_messages(normalize_native_messages(messages))
+    return tuple(sorted(set(chain.from_iterable(_message_labels(message) for message in normalized))))
+
+
+def unconvertible_blocks_error(
+    messages: Sequence[Mapping[str, object]] | None, messages_format: TokenCountMessagesFormat
+) -> str | None:
+    match messages_format:
+        case "openai":
+            return None
+        case "anthropic":
+            labels: Final = unconvertible_anthropic_blocks(messages or ())
+            return (
+                "count_tokens can't convert these content blocks for this model, so the count would leave them out: "
+                + ", ".join(labels)
+                if labels
+                else None
+            )
+        case _:
+            assert_never(messages_format)

litellm/llms/anthropic/pass_through/messages/transformation.py

@@ -63,6 +63,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     def _resolved_provider(self) -> str:
         return self.custom_llm_provider or "anthropic"
 
+    @property
+    def _carries_thinking_block_binding(self) -> bool:
+        return type(self) is AnthropicMessagesConfig
+
     def get_supported_anthropic_messages_params(self, model: str) -> list:
         return [
             "messages",
@@ -514,6 +518,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             model=model,
             optional_params=anthropic_messages_optional_request_params,
             custom_llm_provider=self._resolved_provider,
+            carry_block_binding=self._carries_thinking_block_binding,
         )
 
         self._translate_adaptive_effort_for_non_adaptive_model(

litellm/llms/anthropic/pass_through/messages/utils.py

@@ -119,6 +119,19 @@ def anthropic_system_to_openai_message(system: object) -> ChatCompletionSystemMe
     return ChatCompletionSystemMessage(role="system", content=system)
 
 
+def normalize_native_messages(messages: Sequence[object]) -> list[dict[str, object]]:
+    """Apply the block rewrites every /v1/messages request gets before it reaches a provider."""
+    from litellm.llms.anthropic.common_utils import (
+        flatten_unencrypted_web_search_results_in_anthropic_messages,
+        sanitize_tool_use_ids_in_anthropic_messages,
+        strip_empty_content_blocks_from_anthropic_messages,
+    )
+
+    return flatten_unencrypted_web_search_results_in_anthropic_messages(
+        sanitize_tool_use_ids_in_anthropic_messages(strip_empty_content_blocks_from_anthropic_messages(messages))
+    )
+
+
 @lru_cache(maxsize=1)
 def _anthropic_messages_optional_param_keys() -> frozenset[str]:
     """

litellm/llms/anthropic/pass_through/responses_adapters/transformation.py

@@ -495,6 +495,19 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             result["summary"] = "detailed"
         return result
 
+    @staticmethod
+    def system_blocks_to_instructions(system: object) -> str:
+        if not isinstance(system, list):
+            return ""
+        return "\n".join(
+            text
+            for block in system
+            if isinstance(block, dict)
+            and block.get("type") == "text"
+            and isinstance(text := block.get("text"), str)
+            and text
+        )
+
     def translate_request(
         self,
         anthropic_request: AnthropicMessagesRequest,
@@ -544,9 +557,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             if isinstance(system, str):
                 responses_kwargs["instructions"] = system
             elif isinstance(system, list):
-                responses_kwargs["instructions"] = "\n".join(
+                responses_kwargs["instructions"] = self.system_blocks_to_instructions(system)
-                    filter(None, (b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text"))
-                )
 
         # max_tokens -> max_output_tokens
         max_tokens: Final = anthropic_request.get("max_tokens")

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -28,7 +28,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
         self,
         model: str,
         messages: list[dict[str, Any]],
-        api_key: str,
+        api_key: str | None,
         api_base: str,
         litellm_params: dict[str, Any] | None = None,
         timeout: float | httpx.Timeout | None = None,
@@ -78,6 +78,15 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
                 api_key=api_key,
                 litellm_params=litellm_params,
             )
+            if not {"x-api-key", "Authorization"} & headers.keys():
+                raise AnthropicError(
+                    status_code=401,
+                    message=(
+                        f"No Azure AI API key or Entra ID credential is set for model {model}, so tokens can't be "
+                        "counted. Set api_key or an Entra ID credential on the deployment, or AZURE_AI_API_KEY or "
+                        "AZURE_API_KEY in the proxy environment."
+                    ),
+                )
 
             # Use LiteLLM's async httpx client
             async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.AZURE_AI)

litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py

@@ -5,17 +5,35 @@ Azure AI Anthropic Token Counter implementation using the CountTokens API.
 import os
 from typing import Any, Final
 
+from pydantic import JsonValue
+
+import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
     AzureAIAnthropicCountTokensHandler,
 )
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
-from litellm.types.utils import LlmProviders, TokenCountResponse
+from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
 
 # Global handler instance - reuse across all token counting requests
 azure_ai_anthropic_count_tokens_handler: Final = AzureAIAnthropicCountTokensHandler()
 
 
+def _missing_api_base_response(request_model: str, model_to_use: str) -> TokenCountResponse:
+    return TokenCountResponse(
+        total_tokens=0,
+        request_model=request_model,
+        model_used=model_to_use,
+        tokenizer_type="azure_ai_anthropic_api",
+        error=True,
+        error_message=(
+            f"No Azure AI API base is set for model {request_model or model_to_use}, so tokens can't be counted. "
+            "Set api_base on the deployment or AZURE_AI_API_BASE or AZURE_API_BASE in the proxy environment."
+        ),
+        status_code=500,
+    )
+
+
 class AzureAIAnthropicTokenCounter(BaseTokenCounter):
     """Token counter implementation for Azure AI Anthropic provider using the CountTokens API."""
 
@@ -34,6 +52,8 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
         request_model: str = "",
         tools: list[dict[str, object]] | None = None,
         system: object | None = None,
+        messages_format: TokenCountMessagesFormat = "anthropic",
+        output_format: JsonValue = None,
     ) -> TokenCountResponse | None:
         """
         Count tokens using Azure AI Anthropic's CountTokens API.
@@ -61,18 +81,15 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
         if not api_key:
             api_key = os.getenv("AZURE_AI_API_KEY")
 
-        # Get API base from deployment config or environment
+        api_base: Final = (
-        api_base = litellm_params.get("api_base")
+            litellm_params.get("api_base")
-        if not api_base:
+            or os.getenv("AZURE_AI_API_BASE")
-            api_base = os.getenv("AZURE_AI_API_BASE")
+            or os.getenv("AZURE_API_BASE")
-
+            or litellm.api_base
-        if not api_key:
+        )
-            verbose_logger.warning("No Azure AI API key found for token counting")
-            return None
 
         if not api_base:
-            verbose_logger.warning("No Azure AI API base found for token counting")
+            return _missing_api_base_response(request_model, model_to_use)
-            return None
 
         try:
             result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(

litellm/llms/azure_ai/anthropic/count_tokens/transformation.py

@@ -24,7 +24,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
 
     def get_required_headers(
         self,
-        api_key: str,
+        api_key: str | None,
         litellm_params: dict[str, object] | None = None,
     ) -> dict[str, str]:
         """
@@ -45,12 +45,12 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
             "Content-Type": "application/json",
             "anthropic-version": "2023-06-01",
             "anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
-            "x-api-key": api_key,  # Azure AI Anthropic requires this header
+            **({"x-api-key": api_key} if api_key else {}),
         }
 
         # Also set up Azure auth headers for flexibility
         litellm_params = litellm_params or {}
-        if "api_key" not in litellm_params:
+        if api_key and "api_key" not in litellm_params:
             litellm_params["api_key"] = api_key
 
         litellm_params_obj: Final = GenericLiteLLMParams.model_validate(litellm_params)
@@ -58,8 +58,9 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
         # Get Azure auth headers (api-key or Authorization)
         azure_headers = BaseAzureLLM._base_validate_azure_environment(headers={}, litellm_params=litellm_params_obj)
 
-        # Merge Azure auth headers
         headers.update(azure_headers)
+        if "api-key" in azure_headers:
+            headers.setdefault("x-api-key", azure_headers["api-key"])
 
         return headers
 

litellm/llms/azure_ai/anthropic/messages_transformation.py

@@ -22,6 +22,10 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
     def custom_llm_provider(self) -> str | None:
         return "azure_ai"
 
+    @property
+    def _carries_thinking_block_binding(self) -> bool:
+        return type(self) is AzureAnthropicMessagesConfig
+
     def should_strip_billing_metadata(self) -> bool:
         return True
 

litellm/llms/azure_ai/anthropic/transformation.py

@@ -39,6 +39,10 @@ class AzureAnthropicConfig(AnthropicConfig):
     def custom_llm_provider(self) -> str | None:
         return "azure_ai"
 
+    @property
+    def _carries_thinking_block_binding(self) -> bool:
+        return type(self) is AzureAnthropicConfig
+
     def should_strip_billing_metadata(self) -> bool:
         return True
 

litellm/llms/azure_ai/decisions/transformation.py

@@ -0,0 +1,24 @@
+import re
+from typing import Final
+
+import httpx
+
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+
+_FOUNDRY_ROUTE_SUFFIX: Final = re.compile(r"(/api/projects/[^/]+)?(/openai/v1|/openai|/models|/v1)?$")
+
+
+class AzureAIDecisionsConfig(BaseDecisionsConfig):
+    """Microsoft Foundry decision models (Microsoft-Decision-1), where model is the deployment name"""
+
+    path = "/providers/microsoft/v1/systemone"
+    api_key_env = ("AZURE_AI_API_KEY",)
+    api_base_env = ("AZURE_AI_API_BASE",)
+
+    def missing_api_base_message(self, custom_llm_provider: str) -> str:
+        return "Missing AZURE_AI_API_BASE - set AZURE_AI_API_BASE or pass api_base, e.g. https://<resource>.services.ai.azure.com"
+
+    def get_complete_url(self, api_base: str, model: str) -> str:
+        url: Final = httpx.URL(api_base)
+        resource_path: Final = _FOUNDRY_ROUTE_SUFFIX.sub("", url.path.rstrip("/"))
+        return str(url.copy_with(path=f"{resource_path}{self.path}", query=None, fragment=None))

litellm/llms/base_llm/anthropic_messages/transformation.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from collections.abc import AsyncIterator
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Final
 
 import httpx
 
@@ -156,16 +156,28 @@ class BaseAnthropicMessagesConfig(ABC):
         """
         return 2
 
-    def should_retry_anthropic_messages_on_http_error(self, e: httpx.HTTPStatusError, litellm_params: dict) -> bool:
+    def should_retry_anthropic_messages_on_http_error(
+        self, e: httpx.HTTPStatusError, litellm_params: dict, request_body: object = None
+    ) -> bool:
         """
         When True, async_anthropic_messages_handler will transform the request body
         and issue one more attempt (bounded by max_retry_on_anthropic_messages_http_error).
+        A binding-mismatch 400 is not retried when the body's ``thinking.block_binding``
+        asked for it, since the retry would strip the very blocks the caller pinned.
         """
         from litellm.llms.anthropic.common_utils import (
+            anthropic_request_errors_on_thinking_binding_mismatch,
             is_anthropic_invalid_thinking_block_error,
+            is_anthropic_thinking_binding_mismatch_error,
         )
 
-        return e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text)
+        if e.response.status_code != 400:
+            return False
+        error_text: Final[str] = e.response.text
+        caller_asked_for_this_error: Final = anthropic_request_errors_on_thinking_binding_mismatch(
+            request_body
+        ) and is_anthropic_thinking_binding_mismatch_error(error_text)
+        return not caller_asked_for_this_error and is_anthropic_invalid_thinking_block_error(error_text)
 
     def transform_anthropic_messages_request_on_http_error(self, e: httpx.HTTPStatusError, request_data: dict) -> dict:
         """

litellm/llms/base_llm/base_utils.py

@@ -8,11 +8,11 @@ from abc import ABC, abstractmethod
 from typing import Any, Final
 
 from openai.lib import _parsing, _pydantic
-from pydantic import BaseModel
+from pydantic import BaseModel, JsonValue
 
 from litellm._logging import verbose_logger
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
-from litellm.types.utils import Message, ProviderSpecificModelInfo, TokenCountResponse
+from litellm.types.utils import Message, ProviderSpecificModelInfo, TokenCountMessagesFormat, TokenCountResponse
 
 
 class BaseTokenCounter(ABC):
@@ -26,6 +26,8 @@ class BaseTokenCounter(ABC):
         request_model: str = "",
         tools: list[dict[str, Any]] | None = None,
         system: Any | None = None,
+        messages_format: TokenCountMessagesFormat = "openai",
+        output_format: JsonValue = None,
     ) -> TokenCountResponse | None:
         pass
 

litellm/llms/base_llm/decisions/__init__.py

@@ -0,0 +1,3 @@
+from .transformation import BaseDecisionsConfig
+
+__all__ = ["BaseDecisionsConfig"]

litellm/llms/base_llm/decisions/transformation.py

@@ -0,0 +1,394 @@
+import json
+from abc import ABC
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
+from typing import Final
+
+import httpx
+from pydantic import TypeAdapter, ValidationError
+from typing_extensions import assert_never
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.decisions import (
+    ChoiceAnswer,
+    ChoiceQuestion,
+    DecisionAnswer,
+    DecisionQuestion,
+    DecisionsIRAnswer,
+    DecisionsIRChoiceAnswer,
+    DecisionsIRChoiceOption,
+    DecisionsIRChoiceProbability,
+    DecisionsIRChoiceQuestion,
+    DecisionsIRMessages,
+    DecisionsIRPredicateAnswer,
+    DecisionsIRPredicateQuestion,
+    DecisionsIRQuestion,
+    DecisionsIRRefusal,
+    DecisionsIRRequest,
+    DecisionsIRResponse,
+    DecisionsIRScoreAnswer,
+    DecisionsIRScoreLevel,
+    DecisionsIRScoreProbability,
+    DecisionsIRScoreQuestion,
+    DecisionsIRState,
+    DecisionsIRUsage,
+    DecisionsJSON,
+    DecisionsRequestBody,
+    DecisionsResponse,
+    DecisionsUsage,
+    NoulAnswer,
+    NoulQuestion,
+    OpenAIDecisionInputImage,
+    OpenAIDecisionInputMessage,
+    OpenAIDecisionInputText,
+    ScoreAnswer,
+    ScoreQuestion,
+    UnsupportedDecisionsRequest,
+    systemone_choice_key,
+)
+
+_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
+_SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse)
+_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"})
+_TEXT_ONLY: Final = UnsupportedDecisionsRequest(
+    reason="input_image content parts are not supported because System One providers accept text input only"
+)
+
+
+def decisions_text(value: DecisionsJSON) -> str:
+    return value if isinstance(value, str) else json.dumps(value)
+
+
+def systemone_keys(questions: Sequence[DecisionsIRQuestion]) -> tuple[str, ...]:
+    names: Final = tuple(question.name for question in questions if question.name is not None)
+    if len(frozenset(names)) == len(questions):
+        return names
+    return tuple(str(index) for index in range(len(questions)))
+
+
+def _ir_question(name: str, question: DecisionQuestion) -> DecisionsIRQuestion:
+    extra: Final = MappingProxyType(question.model_extra or {})
+    match question:
+        case NoulQuestion():
+            return DecisionsIRPredicateQuestion(
+                name=name, instructions=question.instructions, criteria=question.criteria, extra=extra
+            )
+        case ChoiceQuestion():
+            return DecisionsIRChoiceQuestion(
+                name=name,
+                instructions=question.instructions,
+                choices=tuple(
+                    DecisionsIRChoiceOption(value=value, description=description)
+                    for value, description in question.criteria.items()
+                ),
+                extra=extra,
+            )
+        case ScoreQuestion():
+            return DecisionsIRScoreQuestion(
+                name=name,
+                instructions=question.instructions,
+                levels=tuple(
+                    DecisionsIRScoreLevel(label=criterion, description=None) for criterion in question.criteria
+                ),
+                extra=extra,
+            )
+        case _:
+            assert_never(question)
+
+
+def systemone_request_to_ir(request: DecisionsRequestBody) -> DecisionsIRRequest:
+    return DecisionsIRRequest(
+        input=DecisionsIRState(state=request.state),
+        questions=tuple(_ir_question(name, question) for name, question in request.questions.items()),
+    )
+
+
+def _has_image(message: OpenAIDecisionInputMessage) -> bool:
+    return not isinstance(message.content, str) and any(
+        isinstance(part, OpenAIDecisionInputImage) for part in message.content
+    )
+
+
+def _message_text(message: OpenAIDecisionInputMessage) -> str:
+    if isinstance(message.content, str):
+        return message.content
+    return "\n\n".join(part.text for part in message.content if isinstance(part, OpenAIDecisionInputText))
+
+
+def _systemone_state(
+    decision_input: DecisionsIRState | DecisionsIRMessages,
+) -> DecisionsJSON | UnsupportedDecisionsRequest:
+    match decision_input:
+        case DecisionsIRState():
+            return decision_input.state
+        case DecisionsIRMessages():
+            if any(_has_image(message) for message in decision_input.messages):
+                return _TEXT_ONLY
+            return "\n\n".join(_message_text(message) for message in decision_input.messages)
+        case _:
+            assert_never(decision_input)
+
+
+def _optional(key: str, value: object) -> Mapping[str, object]:
+    return {} if value is None else {key: value}
+
+
+def _level_criterion(level: DecisionsIRScoreLevel) -> DecisionsJSON:
+    return level.label if level.description is None else f"{decisions_text(level.label)}: {level.description}"
+
+
+def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
+    match question:
+        case DecisionsIRPredicateQuestion():
+            return {
+                "type": "noul",
+                **_optional("instructions", question.instructions),
+                **_optional("criteria", question.criteria),
+                **question.extra,
+            }
+        case DecisionsIRChoiceQuestion():
+            return {
+                "type": "choice",
+                **_optional("instructions", question.instructions),
+                "criteria": {systemone_choice_key(option.value): option.description for option in question.choices},
+                **question.extra,
+            }
+        case DecisionsIRScoreQuestion():
+            return {
+                "type": "score",
+                **_optional("instructions", question.instructions),
+                "criteria": [_level_criterion(level) for level in question.levels],
+                **question.extra,
+            }
+        case _:
+            assert_never(question)
+
+
+def ir_to_systemone_request(
+    model: str, request: DecisionsIRRequest
+) -> Mapping[str, object] | UnsupportedDecisionsRequest:
+    state: Final = _systemone_state(request.input)
+    if isinstance(state, UnsupportedDecisionsRequest):
+        return state
+    keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
+    return {
+        "model": model,
+        "state": state,
+        "questions": {key: _systemone_question(question) for key, question in keyed_questions},
+    }
+
+
+def _ir_choice_answer(question: DecisionsIRChoiceQuestion, answer: ChoiceAnswer) -> DecisionsIRChoiceAnswer:
+    typed_values: Final = {systemone_choice_key(option.value): option.value for option in question.choices}
+    return DecisionsIRChoiceAnswer(
+        choice=typed_values.get(answer.choice, answer.choice),
+        confidence=answer.confidence,
+        probabilities=tuple(
+            DecisionsIRChoiceProbability(value=typed_values.get(key, key), probability=probability)
+            for key, probability in answer.probabilities.items()
+        ),
+        extra=MappingProxyType(answer.model_extra or {}),
+    )
+
+
+def _ir_score_answer(question: DecisionsIRScoreQuestion, answer: ScoreAnswer) -> DecisionsIRScoreAnswer:
+    return DecisionsIRScoreAnswer(
+        score=answer.score,
+        confidence=answer.confidence,
+        probabilities=tuple(
+            DecisionsIRScoreProbability(
+                value=index,
+                label=answer.legend.get(str(index), level.label),
+                probability=answer.probabilities.get(str(index), 0.0),
+            )
+            for index, level in enumerate(question.levels)
+        ),
+        extra=MappingProxyType(answer.model_extra or {}),
+    )
+
+
+def _ir_answer(question: DecisionsIRQuestion, answer: DecisionAnswer | None) -> DecisionsIRAnswer:
+    match question, answer:
+        case DecisionsIRPredicateQuestion(), NoulAnswer():
+            return DecisionsIRPredicateAnswer(probability=answer.noul, extra=MappingProxyType(answer.model_extra or {}))
+        case DecisionsIRChoiceQuestion(), ChoiceAnswer():
+            return _ir_choice_answer(question, answer)
+        case DecisionsIRScoreQuestion(), ScoreAnswer():
+            return _ir_score_answer(question, answer)
+        case _:
+            return DecisionsIRRefusal()
+
+
+def systemone_response_to_ir(response: DecisionsResponse, request: DecisionsIRRequest) -> DecisionsIRResponse:
+    usage: Final = response.usage or DecisionsUsage()
+    keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
+    return DecisionsIRResponse(
+        model=response.model,
+        answers=tuple(_ir_answer(question, response.answers.get(key)) for key, question in keyed_questions),
+        usage=DecisionsIRUsage(
+            input_tokens=usage.input_tokens,
+            output_tokens=usage.output_tokens,
+            cached_tokens=usage.cached_tokens,
+            cache_write_tokens=usage.cache_write_tokens,
+            extra=MappingProxyType(usage.model_extra or {}),
+        ),
+        extra=MappingProxyType(response.model_extra or {}),
+    )
+
+
+def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
+    return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(payload), request)
+
+
+def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None:
+    match answer:
+        case DecisionsIRPredicateAnswer():
+            return NoulAnswer.model_validate({**answer.extra, "type": "noul", "noul": answer.probability})
+        case DecisionsIRChoiceAnswer():
+            return ChoiceAnswer.model_validate(
+                {
+                    **answer.extra,
+                    "type": "choice",
+                    "choice": systemone_choice_key(answer.choice),
+                    "confidence": answer.confidence,
+                    "probabilities": {
+                        systemone_choice_key(item.value): item.probability for item in answer.probabilities
+                    },
+                }
+            )
+        case DecisionsIRScoreAnswer():
+            return ScoreAnswer.model_validate(
+                {
+                    **answer.extra,
+                    "type": "score",
+                    "score": answer.score,
+                    "confidence": answer.confidence,
+                    "legend": {str(item.value): item.label for item in answer.probabilities},
+                    "probabilities": {str(item.value): item.probability for item in answer.probabilities},
+                }
+            )
+        case DecisionsIRRefusal():
+            return None
+        case _:
+            assert_never(answer)
+
+
+def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIRRequest) -> DecisionsResponse:
+    keyed_answers: Final = zip(
+        systemone_keys(request.questions), (_systemone_answer(answer) for answer in response.answers), strict=True
+    )
+    return DecisionsResponse.model_validate(
+        {
+            **response.extra,
+            "model": response.model,
+            "answers": {key: answer for key, answer in keyed_answers if answer is not None},
+            "usage": DecisionsUsage.model_validate(
+                {
+                    **response.usage.extra,
+                    "input_tokens": response.usage.input_tokens,
+                    "output_tokens": response.usage.output_tokens,
+                    "cached_tokens": response.usage.cached_tokens,
+                    "cache_write_tokens": response.usage.cache_write_tokens,
+                }
+            ),
+        }
+    )
+
+
+class BaseDecisionsConfig(ABC):
+    path: str = "/v1/systemone"
+    api_key_env: tuple[str, ...] = ()
+    api_base_env: tuple[str, ...] = ()
+    api_key_required: bool = True
+    supports_safety_identifier: bool = False
+    health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
+        {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})}
+    )
+
+    def get_default_api_base(self) -> str | None:
+        return None
+
+    def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
+        return None
+
+    def missing_api_base_message(self, custom_llm_provider: str) -> str:
+        return f"api_base is required for Decisions provider '{custom_llm_provider}'"
+
+    def resolve_api_base(self, api_base: str | None) -> str | None:
+        return api_base or self._first_secret(self.api_base_env) or self.get_default_api_base()
+
+    def resolve_api_key(self, api_key: str | None) -> str | None:
+        return api_key or self._first_secret(self.api_key_env)
+
+    @staticmethod
+    def _first_secret(names: tuple[str, ...]) -> str | None:
+        return next((value for value in (get_secret_str(name) for name in names) if value), None)
+
+    def canonical_model(self, model: str) -> str:
+        return model
+
+    def request_model(self, model: str) -> str:
+        return model
+
+    def validate_environment(self, headers: Mapping[str, str], model: str, api_key: str | None) -> dict[str, str]:
+        return {
+            **{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS},
+            **({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}),
+            "Content-Type": "application/json",
+        }
+
+    def get_complete_url(self, api_base: str, model: str) -> str:
+        return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}"
+
+    def signs_with_aws(self, api_base: str) -> bool:
+        return False
+
+    def sign_request(
+        self,
+        headers: Mapping[str, str],
+        url: str,
+        api_base: str,
+        body: Mapping[str, object],
+        api_key: str | None,
+        litellm_params: Mapping[str, object],
+    ) -> tuple[Mapping[str, str], bytes | None]:
+        return headers, None
+
+    def transform_decisions_request(
+        self,
+        model: str,
+        request: DecisionsIRRequest,
+        custom_llm_provider: str,
+    ) -> Mapping[str, object] | UnsupportedDecisionsRequest:
+        return ir_to_systemone_request(self.request_model(model), request)
+
+    def unwrap_response(self, payload: object) -> object:
+        return payload
+
+    def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
+        return parse_systemone_response(self.unwrap_response(payload), request)
+
+    def transform_decisions_response(
+        self,
+        model: str,
+        custom_llm_provider: str,
+        raw_response: httpx.Response,
+        request: DecisionsIRRequest,
+    ) -> DecisionsIRResponse:
+        payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content)
+        try:
+            return self.parse_response(payload, request)
+        except ValidationError as error:
+            raise BaseLLMException(
+                status_code=500,
+                message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: {error}",
+            ) from error
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict[str, str] | httpx.Headers,
+    ) -> BaseLLMException:
+        return BaseLLMException(status_code=status_code, message=error_message, headers=headers)

litellm/llms/bedrock/chat/chat_completions/transformation.py

@@ -0,0 +1,514 @@
+"""
+Native OpenAI Chat Completions on Amazon Bedrock Runtime.
+
+AWS serves this surface at
+``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions``
+for Grok 4.6, gpt-oss and GPT 5.6 and newer. GPT 5.6 and newer take it by default
+(``bedrock_runtime_chat_completions_is_default`` in ``common_utils``), so their chat
+completions stay chat completions instead of being rewritten to Converse; the
+``chat_completions/`` route prefix opts any other model in, and ``converse/`` pins a
+model to Converse.
+
+Usage: model="bedrock/global.openai.gpt-6-sol" or
+model="bedrock/chat_completions/openai.gpt-oss-20b-1:0". A request that needs a
+Converse-only feature (``bedrock_request_needs_converse`` in ``common_utils``) is
+still served by Converse.
+"""
+
+from collections.abc import AsyncIterator, Iterator, Mapping
+from dataclasses import dataclass, replace
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal
+
+import httpx
+from pydantic import TypeAdapter
+from typing_extensions import assert_never
+
+import litellm
+from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
+from litellm.litellm_core_utils.prompt_templates.image_handling import (
+    async_inline_remote_media,
+    inline_remote_image_urls,
+    inline_remote_media,
+)
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
+from litellm.llms.bedrock.common_utils import (
+    BedrockError,
+    bedrock_model_is_openai_gpt,
+    split_bedrock_region_path,
+)
+from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
+from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import Choices, ModelResponse, ModelResponseStream
+
+if TYPE_CHECKING:
+    import tiktoken
+
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+REASONING_OPEN_TAG: Final = "<reasoning>"
+REASONING_CLOSE_TAG: Final = "</reasoning>"
+
+_PARAMS_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
+_PARAMS_LIST_ADAPTER: Final = TypeAdapter(list[str])
+
+CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY: Final = MappingProxyType(
+    {
+        "openai.gpt-oss": frozenset(("logit_bias",)),
+        "xai.": frozenset(("frequency_penalty", "presence_penalty")),
+    }
+)
+GPT_CHAT_COMPLETIONS_PARAMS_REFUSED_WHILE_REASONING: Final = frozenset(
+    ("temperature", "top_p", "frequency_penalty", "presence_penalty", "logprobs", "top_logprobs")
+)
+
+
+def chat_completions_params_refused_for(model: str) -> frozenset[str]:
+    """The OpenAI params AWS's Chat Completions endpoint rejects for this model whatever else the request says.
+
+    GPT-OSS answers ``logit_bias`` with a 400 and Grok answers the penalties with a 503, so the native config leaves
+    them out of its supported params and litellm refuses them, or drops them under ``drop_params``, before sending.
+    """
+    model_id: Final = split_bedrock_region_path(model)[1]
+    return frozenset().union(
+        *(refused for family, refused in CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY.items() if family in model_id)
+    )
+
+
+def chat_completions_params_refused_while_reasoning(model: str, params: Mapping[str, object]) -> frozenset[str]:
+    """The params of this request that AWS ties to ``reasoning_effort: "none"`` on the GPT-5.x and GPT-6.x families.
+
+    AWS answers ``temperature``, ``top_p``, the penalties, and logprobs with a 400 while the model reasons, which
+    is every effort but ``"none"`` and the default when none is set, and accepts all of them under ``"none"``.
+    """
+    if params.get("reasoning_effort") == "none" or not bedrock_model_is_openai_gpt(model):
+        return frozenset()
+    return GPT_CHAT_COMPLETIONS_PARAMS_REFUSED_WHILE_REASONING & frozenset(params)
+
+
+def _without_params(params: Mapping[str, object], dropped: frozenset[str]) -> Mapping[str, object]:
+    return MappingProxyType({key: value for key, value in params.items() if key not in dropped})
+
+
+CHAT_COMPLETIONS_REFUSED_REASONING_EFFORTS_BY_FAMILY: Final = MappingProxyType({"xai.": frozenset(("none",))})
+
+
+def chat_completions_reasoning_efforts_refused_for(model: str) -> frozenset[str]:
+    """The ``reasoning_effort`` values AWS's Chat Completions endpoint rejects for this model.
+
+    Grok answers ``"none"`` with a 400 (it takes low, medium, high, and xhigh) where Converse dropped every
+    ``reasoning_effort`` for it, so the native config drops the value and AWS applies its default effort as before.
+    """
+    model_id: Final = split_bedrock_region_path(model)[1]
+    return frozenset().union(
+        *(
+            refused
+            for family, refused in CHAT_COMPLETIONS_REFUSED_REASONING_EFFORTS_BY_FAMILY.items()
+            if family in model_id
+        )
+    )
+
+
+def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) -> Mapping[str, object]:
+    effort: Final = params.get("reasoning_effort")
+    if not isinstance(effort, str) or effort not in chat_completions_reasoning_efforts_refused_for(model):
+        return params
+    return _without_params(params, frozenset(("reasoning_effort",)))
+
+
+def non_string_reasoning_effort(params: Mapping[str, object]) -> frozenset[str]:
+    """``reasoning_effort`` when the request sends it as anything but a string (an int, a list, an object).
+
+    AWS's Chat Completions endpoint answers such a value with a 400 where Converse silently dropped it, so the
+    native config refuses it before the call, or drops it under ``drop_params`` so AWS applies its default effort.
+    """
+    effort: Final = params.get("reasoning_effort")
+    if effort is None or isinstance(effort, str):
+        return frozenset()
+    return frozenset(("reasoning_effort",))
+
+
+def _held_close_tag_prefix(text: str) -> int:
+    return next(
+        (
+            size
+            for size in range(min(len(text), len(REASONING_CLOSE_TAG) - 1), 0, -1)
+            if REASONING_CLOSE_TAG.startswith(text[-size:])
+        ),
+        0,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class ReasoningTagSplitter:
+    """
+    The same split for a stream of content deltas, where a tag can arrive across chunks.
+
+    ``feed`` returns the next state plus the reasoning and content text the delta contributes;
+    ``flush`` releases what the stream ended on before a tag resolved.
+    """
+
+    phase: Literal["start", "reasoning", "after_close", "content"] = "start"
+    pending: str = ""
+
+    def feed(self, text: str) -> tuple["ReasoningTagSplitter", str, str]:
+        match self.phase:
+            case "content":
+                return self, "", text
+            case "after_close":
+                content: Final = text.lstrip()
+                return (replace(self, phase="content") if content else self), "", content
+            case "start":
+                return self._feed_start(self.pending + text)
+            case "reasoning":
+                return self._feed_reasoning(self.pending + text)
+            case _:
+                assert_never(self.phase)
+
+    def _feed_start(self, buffered: str) -> tuple["ReasoningTagSplitter", str, str]:
+        if buffered.startswith(REASONING_OPEN_TAG):
+            return replace(self, phase="reasoning", pending="")._feed_reasoning(buffered[len(REASONING_OPEN_TAG) :])
+        if REASONING_OPEN_TAG.startswith(buffered):
+            return replace(self, pending=buffered), "", ""
+        return replace(self, phase="content", pending=""), "", buffered
+
+    def _feed_reasoning(self, buffered: str) -> tuple["ReasoningTagSplitter", str, str]:
+        close_at: Final = buffered.find(REASONING_CLOSE_TAG)
+        if close_at >= 0:
+            after_close: Final = replace(self, phase="after_close", pending="")
+            next_state, _, content = after_close.feed(buffered[close_at + len(REASONING_CLOSE_TAG) :])
+            return next_state, buffered[:close_at], content
+        held: Final = _held_close_tag_prefix(buffered)
+        return replace(self, pending=buffered[len(buffered) - held :]), buffered[: len(buffered) - held], ""
+
+    def flush(self) -> tuple["ReasoningTagSplitter", str, str]:
+        drained: Final = replace(self, phase="content", pending="")
+        if self.phase == "reasoning":
+            return drained, self.pending, ""
+        return drained, "", self.pending
+
+
+def _split_streamed_content(
+    splitter: ReasoningTagSplitter, content: str | None, finished: bool
+) -> tuple[ReasoningTagSplitter, str, str]:
+    fed_state, fed_reasoning, fed_content = splitter.feed(content or "")
+    if not finished:
+        return fed_state, fed_reasoning, fed_content
+    drained, flushed_reasoning, flushed_content = fed_state.flush()
+    return drained, fed_reasoning + flushed_reasoning, fed_content + flushed_content
+
+
+def split_reasoning_tag(content: str) -> tuple[str | None, str]:
+    """
+    Split gpt-oss's inline ``<reasoning>...</reasoning>`` prefix out of a complete message.
+
+    Runs the streaming splitter over the whole message, so a streamed and a non-streamed
+    response to the same completion split identically. Returns ``(None, content)`` when the
+    message does not start with the tag.
+    """
+    _, reasoning, body = _split_streamed_content(ReasoningTagSplitter(), content, finished=True)
+    return reasoning or None, body
+
+
+class BedrockRuntimeChatCompletionsStreamingHandler(OpenAIChatCompletionStreamingHandler):
+    """OpenAI chunk parsing plus the ``<reasoning>`` split, tracked per choice index."""
+
+    def __init__(
+        self,
+        streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
+        sync_stream: bool,
+        json_mode: bool | None = False,
+    ) -> None:
+        super().__init__(streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode)
+        self._splitters: Mapping[int, ReasoningTagSplitter] = MappingProxyType({})
+
+    def chunk_parser(self, chunk: dict) -> ModelResponseStream:  # mutable-ok: BaseModelResponseIterator signature
+        parsed: Final = super().chunk_parser(chunk)
+        for choice in parsed.choices:
+            next_state, reasoning, content = _split_streamed_content(
+                self._splitters.get(choice.index, ReasoningTagSplitter()),
+                choice.delta.content,
+                choice.finish_reason is not None,
+            )
+            self._splitters = MappingProxyType({**self._splitters, choice.index: next_state})
+            if reasoning:
+                choice.delta.reasoning_content = f"{getattr(choice.delta, 'reasoning_content', None) or ''}{reasoning}"
+            if content or choice.delta.content is not None:
+                choice.delta.content = content
+        return parsed
+
+
+def with_max_completion_tokens(params: Mapping[str, object]) -> Mapping[str, object]:
+    """
+    Send the caller's ``max_tokens`` as ``max_completion_tokens``.
+
+    Every model on this surface accepts ``max_completion_tokens`` and the GPT-5.6 family
+    rejects ``max_tokens``; an explicit ``max_completion_tokens`` wins when both are set.
+    """
+    if "max_tokens" not in params:
+        return params
+    return MappingProxyType(
+        {
+            key: value
+            for key, value in (("max_completion_tokens", params["max_tokens"]), *params.items())
+            if key != "max_tokens"
+        }
+    )
+
+
+class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
+    def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None:
+        super().__init__()
+        self._aws_signer: Final = aws_signer or BaseAWSLLM()
+
+    @property
+    def custom_llm_provider(self) -> str | None:
+        return "bedrock"
+
+    @property
+    def uses_async_transform_request(self) -> bool:
+        return True
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict[str, object] | httpx.Headers,  # mutable-ok: BaseConfig signature
+    ) -> BaseLLMException:
+        return BedrockError(status_code=status_code, message=error_message, headers=headers)
+
+    def validate_environment(
+        self,
+        headers: dict,  # mutable-ok: BaseConfig signature
+        model: str,
+        messages: list[AllMessageValues],
+        optional_params: dict,  # mutable-ok: BaseConfig signature
+        litellm_params: dict,  # mutable-ok: BaseConfig signature
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict:  # mutable-ok: BaseConfig signature
+        return super().validate_environment(
+            headers=headers,
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            api_key=bedrock_bearer_token(api_key),
+            api_base=api_base,
+        )
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: dict,  # mutable-ok: BaseConfig signature
+        litellm_params: dict,  # mutable-ok: BaseConfig signature
+        stream: bool | None = None,
+    ) -> str:
+        if api_base is not None and "chat/completions" in api_base:
+            return api_base.rstrip("/")
+        aws_region_name: Final = self._aws_signer._get_aws_region_name(  # pyright: ignore[reportPrivateUsage]  # BaseAWSLLM has no public region resolver
+            optional_params=self._params_with_region_from_path(optional_params, model), model=model
+        )
+        configured_runtime_endpoint: Final = optional_params.get("aws_bedrock_runtime_endpoint")
+        _, proxy_endpoint_url = self._aws_signer.get_runtime_endpoint(
+            api_base=api_base,
+            aws_bedrock_runtime_endpoint=(
+                configured_runtime_endpoint if isinstance(configured_runtime_endpoint, str) else None
+            ),
+            aws_region_name=aws_region_name,
+        )
+        base: Final = proxy_endpoint_url.rstrip("/")
+        if base.endswith("/openai/v1/chat/completions"):
+            return base
+        if base.endswith("/openai/v1"):
+            return f"{base}/chat/completions"
+        return f"{base}/openai/v1/chat/completions"
+
+    def _params_with_region_from_path(
+        self, optional_params: dict, model: str | None
+    ) -> dict:  # mutable-ok: BaseAWSLLM's region resolver and signer take a plain dict
+        region_from_path, _ = split_bedrock_region_path(model or "")
+        if region_from_path is None or optional_params.get("aws_region_name") is not None:
+            return optional_params
+        return {**optional_params, "aws_region_name": region_from_path}
+
+    def sign_request(
+        self,
+        headers: dict,  # mutable-ok: BaseConfig signature
+        optional_params: dict,  # mutable-ok: BaseConfig signature
+        request_data: dict,  # mutable-ok: BaseConfig signature
+        api_base: str,
+        api_key: str | None = None,
+        model: str | None = None,
+        stream: bool | None = None,
+        fake_stream: bool | None = None,
+    ) -> tuple[dict, bytes | None]:  # mutable-ok: BaseConfig signature
+        return self._aws_signer._sign_request(  # pyright: ignore[reportPrivateUsage]  # BaseAWSLLM has no public signer
+            service_name="bedrock",
+            headers=headers,
+            optional_params=self._params_with_region_from_path(optional_params, model),
+            request_data=request_data,
+            api_base=api_base,
+            api_key=api_key,
+            model=model,
+            stream=stream,
+            fake_stream=fake_stream,
+        )
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,  # mutable-ok: BaseConfig signature
+        optional_params: dict,  # mutable-ok: BaseConfig signature
+        model: str,
+        drop_params: bool,
+        replace_max_completion_tokens_with_max_tokens: bool = False,
+    ) -> dict:  # mutable-ok: BaseConfig signature
+        mapped: Final = _PARAMS_DICT_ADAPTER.validate_python(
+            super().map_openai_params(
+                non_default_params=non_default_params,
+                optional_params=optional_params,
+                model=model,
+                drop_params=drop_params,
+                replace_max_completion_tokens_with_max_tokens=replace_max_completion_tokens_with_max_tokens,
+            )
+        )
+        raw_params: Final = _PARAMS_DICT_ADAPTER.validate_python(non_default_params)
+        malformed_effort: Final = non_string_reasoning_effort(raw_params)
+        refused_while_reasoning: Final = chat_completions_params_refused_while_reasoning(model, raw_params)
+        if malformed_effort and not (litellm.drop_params or drop_params):
+            raise litellm.utils.UnsupportedParamsError(
+                message=(
+                    f"{model} takes reasoning_effort as a string on Bedrock's Chat Completions endpoint, not "
+                    f"{type(raw_params['reasoning_effort']).__name__}. Send one of its named efforts, or "
+                    "set `litellm.drop_params = True` to drop it"
+                ),
+                status_code=400,
+            )
+        if refused_while_reasoning and not (litellm.drop_params or drop_params):
+            raise litellm.utils.UnsupportedParamsError(
+                message=(
+                    f"{model} doesn't support {sorted(refused_while_reasoning)} while reasoning is active on "
+                    "Bedrock's Chat Completions endpoint. Set reasoning_effort to 'none' to send them, or set "
+                    "`litellm.drop_params = True` to drop them"
+                ),
+                status_code=400,
+            )
+        return dict(
+            without_refused_reasoning_effort(
+                model,
+                with_max_completion_tokens(_without_params(mapped, refused_while_reasoning | malformed_effort)),
+            )
+        )
+
+    def _inference_params(
+        self, optional_params: Mapping[str, object]
+    ) -> dict[str, object]:  # mutable-ok: BaseConfig signature of transform_request
+        return {
+            key: value
+            for key, value in optional_params.items()
+            if key not in self._aws_signer.aws_authentication_params
+        }
+
+    def transform_request(
+        self,
+        model: str,
+        messages: list[AllMessageValues],  # mutable-ok: BaseConfig signature
+        optional_params: dict,  # mutable-ok: BaseConfig signature
+        litellm_params: dict,  # mutable-ok: BaseConfig signature
+        headers: dict,  # mutable-ok: BaseConfig signature
+    ) -> dict:  # mutable-ok: BaseConfig signature
+        optional_params_view: Final = _PARAMS_DICT_ADAPTER.validate_python(optional_params)
+        return super().transform_request(
+            model=split_bedrock_region_path(model)[1],
+            messages=inline_remote_media(messages, should_inline=inline_remote_image_urls),
+            optional_params=self._inference_params(optional_params_view),
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
+    async def async_transform_request(
+        self,
+        model: str,
+        messages: list[AllMessageValues],  # mutable-ok: BaseConfig signature
+        optional_params: dict,  # mutable-ok: BaseConfig signature
+        litellm_params: dict,  # mutable-ok: BaseConfig signature
+        headers: dict,  # mutable-ok: BaseConfig signature
+    ) -> dict:  # mutable-ok: BaseConfig signature
+        optional_params_view: Final = _PARAMS_DICT_ADAPTER.validate_python(optional_params)
+        return await super().async_transform_request(
+            model=split_bedrock_region_path(model)[1],
+            messages=await async_inline_remote_media(messages, should_inline=inline_remote_image_urls),
+            optional_params=self._inference_params(optional_params_view),
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
+    def transform_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        model_response: ModelResponse,
+        logging_obj: "LiteLLMLoggingObj",
+        request_data: dict,  # mutable-ok: BaseConfig signature
+        messages: list[AllMessageValues],  # mutable-ok: BaseConfig signature
+        optional_params: dict,  # mutable-ok: BaseConfig signature
+        litellm_params: dict,  # mutable-ok: BaseConfig signature
+        encoding: "tiktoken.Encoding | None",
+        api_key: str | None = None,
+        json_mode: bool | None = None,
+    ) -> ModelResponse:
+        response: Final = super().transform_response(
+            model=model,
+            raw_response=raw_response,
+            model_response=model_response,
+            logging_obj=logging_obj,
+            request_data=request_data,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            encoding=encoding,
+            api_key=api_key,
+            json_mode=json_mode,
+        )
+        set_provider_response_headers_in_hidden_params(response, raw_response.headers)
+        for choice in response.choices:
+            if not isinstance(choice, Choices) or not isinstance(choice.message.content, str):
+                continue
+            reasoning, content = split_reasoning_tag(choice.message.content)
+            if reasoning is not None:
+                choice.message.reasoning_content = (
+                    f"{getattr(choice.message, 'reasoning_content', None) or ''}{reasoning}"
+                )
+            choice.message.content = content
+        return response
+
+    def get_supported_openai_params(self, model: str) -> list:  # mutable-ok: BaseConfig signature
+        refused: Final = frozenset(("n", *chat_completions_params_refused_for(model)))
+        base_params: Final = tuple(
+            param
+            for param in _PARAMS_LIST_ADAPTER.validate_python(super().get_supported_openai_params(model))
+            if param not in refused
+        )
+        reasoning_param: Final = (
+            ("reasoning_effort",)
+            if "reasoning_effort" not in base_params
+            and litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider)
+            else ()
+        )
+        return [*base_params, *reasoning_param]
+
+    def get_model_response_iterator(
+        self,
+        streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
+        sync_stream: bool,
+        json_mode: bool | None = False,
+    ) -> BedrockRuntimeChatCompletionsStreamingHandler:
+        return BedrockRuntimeChatCompletionsStreamingHandler(
+            streaming_response=streaming_response,
+            sync_stream=sync_stream,
+            json_mode=json_mode,
+        )

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py

@@ -12,7 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
     convert_url_to_base64,
 )
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
-from litellm.llms.anthropic.common_utils import AnthropicModelInfo
+from litellm.llms.anthropic.common_utils import AnthropicModelInfo, thinking_block_binding
 from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
     AmazonInvokeConfig,
 )
@@ -67,6 +67,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
     def custom_llm_provider(self) -> str | None:
         return "bedrock"
 
+    @property
+    def _carries_thinking_block_binding(self) -> bool:
+        return type(self) is AmazonAnthropicClaudeConfig
+
     def should_strip_billing_metadata(self) -> bool:
         return True
 
@@ -115,7 +119,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         model = original_model
 
         AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
-            model=original_model, optional_params=optional_params, custom_llm_provider="bedrock"
+            model=original_model,
+            optional_params=optional_params,
+            custom_llm_provider="bedrock",
+            carry_block_binding=self._carries_thinking_block_binding,
         )
         translated_thinking: Final = optional_params.get("thinking")
         if (
@@ -123,8 +130,11 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
             and isinstance(translated_thinking, dict)
             and translated_thinking.get("type") == "adaptive"
         ):
-            thinking_with_display: Final[AnthropicThinkingParam] = {"type": "adaptive", "display": "updates"}
+            optional_params["thinking"] = {
-            optional_params["thinking"] = thinking_with_display
+                **(thinking_block_binding(translated_thinking) if self._carries_thinking_block_binding else {}),
+                "type": "adaptive",
+                "display": "updates",
+            }
 
         # The stub model hides the original model from the parent's forced-tool-use backstop
         response_format_tool_choice: Final = optional_params.get("tool_choice")

litellm/llms/bedrock/common_utils.py

@@ -10,6 +10,7 @@ import json
 import os
 import re
 from collections.abc import Mapping, Sequence
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict
 
 if TYPE_CHECKING:
@@ -28,6 +29,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import (
 )
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned
 from litellm.secret_managers.main import get_secret, get_secret_str
 from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams
 
@@ -37,6 +39,21 @@ if TYPE_CHECKING:
 
 _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
 _OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.")
+_OPENAI_GPT_VERSION_RE: Final = re.compile(r"(^|[./])openai\.gpt-(\d{1,3})(?!\d)(?:\.(\d{1,3})(?!\d))?")
+_BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE: Final = (5, 6)
+_BEDROCK_RUNTIME_CHAT_COMPLETIONS_ENDPOINT: Final = "/v1/chat/completions"
+BedrockRoute = Literal[
+    "converse",
+    "invoke",
+    "claude_platform",
+    "converse_like",
+    "agent",
+    "agentcore",
+    "async_invoke",
+    "openai",
+    "mantle",
+    "chat_completions",
+]
 
 
 def error_response_text(response: httpx.Response) -> str:
@@ -787,12 +804,191 @@ def is_bedrock_application_inference_profile_arn(model: str) -> bool:
 
 def strip_bedrock_routing_prefix(model: str) -> str:
     """Strip LiteLLM routing prefixes from model name."""
-    for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "mantle/", "nova-2/", "nova/"]:
+    for prefix in ["bedrock/", "chat_completions/", "converse/", "invoke/", "openai/", "mantle/", "nova-2/", "nova/"]:
         if model.startswith(prefix):
             model = model.split("/", 1)[1]
     return model
 
 
+BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX: Final = "chat_completions/"
+BEDROCK_CONVERSE_ROUTE_PREFIX: Final = "converse/"
+
+
+def without_bedrock_route_prefix(model: str) -> str:
+    return model.replace(BEDROCK_CONVERSE_ROUTE_PREFIX, "").replace(BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX, "")
+
+
+def split_bedrock_region_path(model: str) -> tuple[str | None, str]:
+    """Split a ``<region>/<model-id>`` routing path into the region and the id AWS receives.
+
+    ``bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0`` -> ``("us-gov-west-1", "openai.gpt-oss-20b-1:0")``;
+    a model without a region path comes back as ``(None, <routing-prefix-stripped id>)``.
+    """
+    stripped: Final = strip_bedrock_routing_prefix(model)
+    region, separator, model_id = stripped.partition("/")
+    if separator and region in _get_all_bedrock_regions():
+        return region, model_id
+    return None, stripped
+
+
+_MODEL_COST_ENTRY_ADAPTER: Final = TypeAdapter(dict[str, object])
+
+
+def _model_cost_entry(key: str) -> Mapping[str, object] | None:
+    raw: Final = litellm.model_cost.get(key)
+    return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
+
+
+def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
+    return tuple(
+        _model_cost_entry(key)
+        for key in (model, strip_bedrock_routing_prefix(model), split_bedrock_region_path(model)[1])
+    )
+
+
+def _bedrock_price_map_flag(model: str, flag: str) -> bool:
+    return any(entry is not None and entry.get(flag) is True for entry in _bedrock_price_map_entries(model))
+
+
+def _price_map_entry_lists_endpoint(entry: Mapping[str, object] | None, endpoint: str) -> bool:
+    endpoints: Final = None if entry is None else entry.get("supported_endpoints")
+    return isinstance(endpoints, (list, tuple)) and endpoint in endpoints
+
+
+def _openai_gpt_version(model: str) -> tuple[int, int] | None:
+    match: Final = _OPENAI_GPT_VERSION_RE.search(model)
+    if match is None:
+        return None
+    return int(match.group(2)), int(match.group(3) or 0)
+
+
+def bedrock_runtime_chat_completions_is_default(model: str) -> bool:
+    """Whether a model with no route prefix goes to bedrock-runtime's native Chat Completions by default.
+
+    GPT 5.6 and newer (``openai.gpt-<major>[.<minor>]`` at or above 5.6, which gpt-oss never matches) whose
+    price-map row lists ``/v1/chat/completions`` in ``supported_endpoints``. Older GPT rows, gpt-oss and Grok
+    stay on Converse unless the ``chat_completions/`` prefix opts them in.
+    """
+    version: Final = _openai_gpt_version(model)
+    if version is None or version < _BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE:
+        return False
+    return any(
+        _price_map_entry_lists_endpoint(entry, _BEDROCK_RUNTIME_CHAT_COMPLETIONS_ENDPOINT)
+        for entry in _bedrock_price_map_entries(model)
+    )
+
+
+def bedrock_runtime_chat_completions_serves_tools_with_reasoning(model: str) -> bool:
+    """Whether AWS's native Chat Completions serves this model's function tools with any ``reasoning_effort``.
+
+    Data-driven from the price-map ``supports_bedrock_runtime_chat_completions_tools_with_reasoning``
+    flag (gpt-oss, Grok). Without it AWS only takes tools with ``reasoning_effort="none"``
+    (the GPT-5.6 family), and Converse serves tools with any effort, so those requests fall back to it.
+    """
+    return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_tools_with_reasoning")
+
+
+def bedrock_runtime_chat_completions_enforces_response_format(model: str) -> bool:
+    """Whether AWS's native Chat Completions enforces a ``response_format`` schema for this model.
+
+    Data-driven from the price-map ``supports_bedrock_runtime_chat_completions_response_format`` flag
+    (GPT-5.6, Grok). Without it AWS accepts the field and answers with unconstrained text (gpt-oss), so
+    Converse, which emulates the schema through a forced ``json_tool_call`` tool, serves those requests.
+    """
+    return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_response_format")
+
+
+def bedrock_model_is_openai_gpt(model: str) -> bool:
+    """A GPT-5.x or GPT-6.x id, never GPT-OSS: the families whose sampling params AWS ties to reasoning being off."""
+    return _openai_gpt_version(model) is not None
+
+
+BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset(
+    (
+        "guardrailConfig",
+        "performanceConfig",
+        "serviceTier",
+        "requestMetadata",
+        "outputConfig",
+        "thinking",
+        "additionalModelRequestFields",
+        "top_k",
+        "stop",
+        "model_id",
+    )
+)
+
+
+def _response_format_needs_converse(model: str, response_format: object) -> bool:
+    if response_format is None:
+        return False
+    if not isinstance(response_format, Mapping):
+        return not bedrock_runtime_chat_completions_enforces_response_format(model)
+    response_format_type: Final = response_format.get("type")
+    if response_format_type == "text":
+        return False
+    is_json_schema: Final = response_format_type == "json_schema" and "json_schema" in response_format
+    return not (is_json_schema and bedrock_runtime_chat_completions_enforces_response_format(model))
+
+
+def bedrock_request_needs_converse(model: str, request_params: Mapping[str, object]) -> bool:
+    """Whether a request on the native Chat Completions route must still be served by Converse.
+
+    The route is the default for GPT 5.6 and newer (``bedrock_runtime_chat_completions_is_default``) and the
+    ``chat_completions/`` prefix's opt-in for the rest; this decides the fallback for both alike.
+
+    Converse-shaped body keys (``BEDROCK_CONVERSE_ONLY_REQUEST_KEYS``, the Anthropic-style ``thinking``
+    block and the ``additionalModelRequestFields`` / ``top_k`` extension params included, which only Converse
+    forwards as ``additionalModelRequestFields`` and ``inferenceConfig``) have no field on
+    AWS's native OpenAI surface, a ``model_id`` override (an application inference profile or provisioned
+    throughput ARN) is only encoded into Converse's request URL and so stays on Converse like the
+    ``bedrock/arn:...`` model form, ``stop`` stays on Converse where it fails loudly instead of silently
+    stopping hidden reasoning, operator-owned request metadata is only written onto the Converse body,
+    function tools (``tools`` or legacy ``functions``) on a model without
+    ``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` are rejected there unless
+    ``reasoning_effort`` is exactly ``"none"``, and a ``response_format`` goes native only as
+    ``{"type": "json_schema", "json_schema": ...}`` (a pydantic model is converted to that) on a model with
+    ``supports_bedrock_runtime_chat_completions_response_format``: a schema on any other model is only
+    honored by Converse, and every ``json_object`` form (``response_schema`` included) keeps Converse's
+    handling everywhere, since AWS's native surface rejects that type with a 400 unless the prompt
+    mentions json.
+    """
+    if any(request_params.get(key) is not None for key in BEDROCK_CONVERSE_ONLY_REQUEST_KEYS):
+        return True
+    if bedrock_request_metadata_is_owned():
+        return True
+    if _response_format_needs_converse(model, request_params.get("response_format")):
+        return True
+    if not (request_params.get("tools") or request_params.get("functions")):
+        return False
+    return (
+        not bedrock_runtime_chat_completions_serves_tools_with_reasoning(model)
+        and request_params.get("reasoning_effort") != "none"
+    )
+
+
+def _chat_completions_unless_converse_needed(
+    model: str, request_params: Mapping[str, object] | None
+) -> Literal["converse", "chat_completions"]:
+    if request_params is not None and bedrock_request_needs_converse(model, request_params):
+        return "converse"
+    return "chat_completions"
+
+
+def bedrock_route_for_request(
+    model: str, request_params: Mapping[str, object], additional_drop_params: Sequence[str] | None
+) -> BedrockRoute:
+    """The route for one request, decided from the caller's raw params before any provider mapping.
+
+    Param mapping and dispatch both call this with the same inputs, so a request that falls back to
+    Converse is mapped with the Converse config and sent to Converse, never one without the other.
+    """
+    dropped: Final = frozenset(additional_drop_params or ())
+    return BedrockModelInfo.get_bedrock_route(
+        model, MappingProxyType({key: value for key, value in request_params.items() if key not in dropped})
+    )
+
+
 def strip_bedrock_throughput_suffix(model: str) -> str:
     """Strip throughput tier suffixes and context window suffixes from Bedrock model names."""
     import re
@@ -1150,19 +1346,16 @@ class BedrockModelInfo(BaseLLMModelInfo):
     @staticmethod
     def get_bedrock_route(
         model: str,
-    ) -> Literal[
+        request_params: Mapping[str, object] | None = None,
-        "converse",
+    ) -> BedrockRoute:
-        "invoke",
-        "claude_platform",
-        "converse_like",
-        "agent",
-        "agentcore",
-        "async_invoke",
-        "openai",
-        "mantle",
-    ]:
         """
         Get the bedrock route for the given model.
+
+        GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by default
+        (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
+        ``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
+        feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
+        model stays on Converse without the prefix.
         """
         route_mappings: dict[
             str,
@@ -1176,6 +1369,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
                 "async_invoke",
                 "openai",
                 "mantle",
+                "chat_completions",
             ],
         ] = {
             "invoke/": "invoke",
@@ -1197,6 +1391,9 @@ class BedrockModelInfo(BaseLLMModelInfo):
             if BedrockModelInfo._model_has_route_prefix(model, prefix):
                 return route_type
 
+        if BedrockModelInfo._model_has_route_prefix(model, "chat_completions/"):
+            return _chat_completions_unless_converse_needed(model, request_params)
+
         # Check for nova spec prefixes (nova/ and nova-2/)
         _model_after_bedrock: Final = model.replace("bedrock/", "", 1)
         if _model_after_bedrock.startswith("nova-2/") or _model_after_bedrock.startswith("nova/"):
@@ -1205,6 +1402,9 @@ class BedrockModelInfo(BaseLLMModelInfo):
         if is_bedrock_application_inference_profile_arn(model):
             return "converse"
 
+        if bedrock_runtime_chat_completions_is_default(model):
+            return _chat_completions_unless_converse_needed(model, request_params)
+
         base_model: Final = BedrockModelInfo.get_base_model(model)
         alt_model: Final = BedrockModelInfo.get_non_litellm_routing_model_name(model=model)
         if base_model in litellm.bedrock_converse_models or alt_model in litellm.bedrock_converse_models:
@@ -1383,6 +1583,8 @@ def get_bedrock_chat_config(model: str):
         return litellm.AmazonConverseConfig()
     elif bedrock_route == "openai":
         return litellm.AmazonBedrockOpenAIConfig()
+    elif bedrock_route == "chat_completions":
+        return litellm.AmazonBedrockRuntimeChatCompletionsConfig()
     elif bedrock_route == "agent":
         from litellm.llms.bedrock.chat.invoke_agent.transformation import (
             AmazonInvokeAgentConfig,

litellm/llms/bedrock/count_tokens/bedrock_token_counter.py

@@ -5,11 +5,13 @@ Bedrock Token Counter implementation using the CountTokens API.
 from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
+from pydantic import JsonValue
+
 from litellm._logging import verbose_logger
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
 from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
 from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
-from litellm.types.utils import LlmProviders, TokenCountResponse
+from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
 
 
 class BedrockTokenCounter(BaseTokenCounter):
@@ -33,6 +35,8 @@ class BedrockTokenCounter(BaseTokenCounter):
         request_model: str = "",
         tools: Sequence[Mapping[str, object]] | None = None,
         system: object | None = None,
+        messages_format: TokenCountMessagesFormat = "anthropic",
+        output_format: JsonValue = None,
     ) -> TokenCountResponse | None:
         """
         Count tokens using AWS Bedrock's CountTokens API.

litellm/llms/bedrock/count_tokens/kimi_k3.py

@@ -0,0 +1,502 @@
+"""
+Exact Bedrock token counts for Kimi K3, which has no CountTokens support.
+
+Renders the Converse body litellm sends through Moonshot's Kimi K3 chat template and counts it with the
+official tokenizer, downloaded once from Hugging Face at a pinned revision and checked against its SHA-256.
+"""
+
+import asyncio
+import base64
+import copy
+import hashlib
+import json
+import threading
+import time
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from concurrent.futures import Future
+from dataclasses import dataclass
+from enum import Enum
+from itertools import chain, groupby
+from pathlib import Path
+from typing import Annotated, Final, Literal, TypeAlias
+
+import regex
+import tiktoken
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError, model_validator
+from typing_extensions import Self
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.prompt_templates.factory import (
+    chat_completion_tools_for,
+    is_flat_responses_tool,
+)
+from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
+from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
+
+KIMI_K3_REPO: Final = "moonshotai/Kimi-K3"
+KIMI_K3_REVISION: Final = "f831ab66814297da540d832a5235f8e904f29d06"
+KIMI_K3_RANKS_FILE: Final = "tiktoken.model"
+KIMI_K3_RANKS_SHA256: Final = "b6c497a7469b33ced9c38afb1ad6e47f03f5e5dc05f15930799210ec050c5103"
+_PATTERN: Final = "|".join(
+    (
+        r"[\p{Han}]+",
+        r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?",
+        r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?",
+        r"\p{N}{1,3}",
+        r" ?[^\s\p{L}\p{N}]+[\r\n]*",
+        r"\s*[\r\n]+",
+        r"\s+(?!\S)",
+        r"\s+",
+    )
+)
+# Ends chunks only where the pattern above always splits, so their counts sum to the whole text's count while BPE
+# stays linear. Only a run of over 1024 characters with no such split, like 2000 spaces, is cut mid-piece.
+_CHUNK: Final = regex.compile(
+    r"(?s).{1,1024}(?:(?<=\S)(?= \S)|(?<=\p{N})(?=\P{N})|(?<=[^\s\p{N}])(?=\p{N})|(?<=\p{L})(?=[^\p{L}\p{M}'])"
+    r"|(?<=[\r\n])(?=\S)|\Z)|.{1024}\p{N}*"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class KimiK3TokenizerSource:
+    ranks_path: Callable[[], str]
+    sha256: str
+
+
+def _hugging_face_ranks_path() -> str:
+    import huggingface_hub
+
+    return huggingface_hub.hf_hub_download(
+        repo_id=KIMI_K3_REPO,
+        filename=KIMI_K3_RANKS_FILE,
+        revision=KIMI_K3_REVISION,
+        local_files_only=litellm.disable_hf_tokenizer_download is True,
+    )
+
+
+HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256)
+
+
+def _loaded(source: KimiK3TokenizerSource) -> tiktoken.Encoding | str:
+    try:
+        data: Final = Path(source.ranks_path()).read_bytes()
+    except Exception as e:  # noqa: BLE001  # a download or read failure of any kind falls back to the estimate
+        return type(e).__name__
+    if hashlib.sha256(data).hexdigest() != source.sha256:
+        return "SHA-256 mismatch"
+    rows: Final = (line.split() for line in data.splitlines() if line)
+    return tiktoken.Encoding(
+        name="kimi_k3",
+        pat_str=_PATTERN,
+        mergeable_ranks={base64.b64decode(token): int(rank) for token, rank in rows},
+        special_tokens={},
+    )
+
+
+def load_kimi_k3_encoding(source: KimiK3TokenizerSource) -> tiktoken.Encoding | None:
+    loaded: Final = _loaded(source)
+    if isinstance(loaded, tiktoken.Encoding):
+        return loaded
+    verbose_logger.warning(
+        "Kimi K3 tokenizer unavailable (%s), so Kimi K3 token counts are estimated until a later count loads it. "
+        "Allow downloads of %s from Hugging Face repo %s at revision %s, or put it in the Hugging Face cache",
+        loaded,
+        KIMI_K3_RANKS_FILE,
+        KIMI_K3_REPO,
+        KIMI_K3_REVISION,
+    )
+    return None
+
+
+def _started_load(source: KimiK3TokenizerSource) -> "Future[tiktoken.Encoding | None]":
+    encoding: Final = Future[tiktoken.Encoding | None]()
+    threading.Thread(
+        target=lambda: encoding.set_result(load_kimi_k3_encoding(source)), name="litellm-kimi-k3-tokenizer", daemon=True
+    ).start()
+    return encoding
+
+
+@dataclass(frozen=True, slots=True)
+class _EncodingLoad:
+    encoding: "Future[tiktoken.Encoding | None]"
+    started_at: float
+    is_retry: bool
+
+    def retry_due(self, now: float, retry_after_seconds: float) -> bool:
+        return self.encoding.done() and self.encoding.result() is None and now - self.started_at >= retry_after_seconds
+
+
+@dataclass(slots=True)
+class KimiK3Encodings:
+    source: KimiK3TokenizerSource
+    retry_after_seconds: float
+    clock: Callable[[], float] = time.monotonic
+    current: _EncodingLoad | None = None
+
+    def _load(self) -> _EncodingLoad:
+        now: Final = self.clock()
+        current: Final = self.current
+        if current is not None and not current.retry_due(now, self.retry_after_seconds):
+            return current
+        load: Final = _EncodingLoad(encoding=_started_load(self.source), started_at=now, is_retry=current is not None)
+        self.current = load
+        return load
+
+    async def encoding(self, wait_seconds: float) -> tiktoken.Encoding | None:
+        load: Final = self._load()
+        remaining: Final = 0 if load.is_retry else load.started_at + wait_seconds - self.clock()
+        if load.encoding.done() or remaining <= 0:
+            return load.encoding.result() if load.encoding.done() else None
+        await asyncio.wait((asyncio.wrap_future(load.encoding),), timeout=remaining)
+        if load.encoding.done():
+            return load.encoding.result()
+        verbose_logger.warning(
+            "Kimi K3 tokenizer still loading after %s s, so Kimi K3 token counts are estimated until it loads. "
+            "Check that %s can be downloaded from Hugging Face repo %s at revision %s, or put it in the Hugging Face cache",
+            wait_seconds,
+            KIMI_K3_RANKS_FILE,
+            KIMI_K3_REPO,
+            KIMI_K3_REVISION,
+        )
+        return None
+
+
+class _Converse(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+
+class _Text(_Converse):
+    text: str
+
+
+class _CachePoint(_Converse):
+    cache_point: JsonValue = Field(alias="cachePoint")
+
+
+class _ToolUse(_Converse):
+    tool_use_id: str = Field(alias="toolUseId")
+    name: str
+    arguments: Mapping[str, JsonValue] = Field(alias="input")
+
+
+class _ToolUseBlock(_Converse):
+    tool_use: _ToolUse = Field(alias="toolUse")
+
+
+class _ToolResult(_Converse):
+    tool_use_id: str = Field(alias="toolUseId")
+    content: tuple[_Text, ...]
+    status: str | None = None
+
+
+class _ToolResultBlock(_Converse):
+    tool_result: _ToolResult = Field(alias="toolResult")
+
+
+class _ReasoningText(_Converse):
+    text: str
+    signature: str | None = None
+
+
+class _Reasoning(_Converse):
+    reasoning_text: _ReasoningText = Field(alias="reasoningText")
+
+
+class _ReasoningBlock(_Converse):
+    reasoning_content: _Reasoning = Field(alias="reasoningContent")
+
+
+class _UserMessage(_Converse):
+    role: Literal["user"]
+    content: tuple[_Text | _ToolResultBlock | _CachePoint, ...]
+
+
+class _AssistantMessage(_Converse):
+    role: Literal["assistant"]
+    content: tuple[_Text | _ToolUseBlock | _ReasoningBlock | _CachePoint, ...]
+
+    def calls(self) -> tuple[_ToolUse, ...]:
+        return tuple(block.tool_use for block in self.content if isinstance(block, _ToolUseBlock))
+
+
+class _InputSchema(_Converse):
+    definition: JsonValue = Field(alias="json")
+
+
+class _ToolSpec(_Converse):
+    name: str
+    description: str | None = None
+    input_schema: _InputSchema = Field(alias="inputSchema")
+
+    def declared(self) -> Mapping[str, JsonValue]:
+        described: Final[Mapping[str, JsonValue]] = (
+            {} if self.description is None else {"description": self.description}
+        )
+        return {"name": self.name, **described, "parameters": self.input_schema.definition}
+
+
+class _ToolSpecBlock(_Converse):
+    tool_spec: _ToolSpec = Field(alias="toolSpec")
+
+
+class _ToolConfig(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+    tools: tuple[_ToolSpecBlock | _CachePoint, ...]
+
+
+_Message: TypeAlias = Annotated[_UserMessage | _AssistantMessage, Field(discriminator="role")]
+
+
+def _result_ids(message: _Message) -> frozenset[str]:
+    if isinstance(message, _AssistantMessage):
+        return frozenset()
+    return frozenset(block.tool_result.tool_use_id for block in message.content if isinstance(block, _ToolResultBlock))
+
+
+def _call_ids(message: _Message | None) -> frozenset[str]:
+    if not isinstance(message, _AssistantMessage):
+        return frozenset()
+    return frozenset(call.tool_use_id for call in message.calls())
+
+
+class _ConverseRequest(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+    messages: tuple[_Message, ...]
+    system: tuple[_Text | _CachePoint, ...] = ()
+    tool_config: _ToolConfig | None = Field(default=None, alias="toolConfig")
+
+    @model_validator(mode="after")
+    def _results_answer_the_previous_calls(self) -> Self:
+        pairs: Final = zip((None, *self.messages), self.messages, strict=False)
+        if not all(_result_ids(message) <= _call_ids(previous) for previous, message in pairs):
+            raise ValueError("a toolResult does not answer a toolUse of the previous assistant message")
+        return self
+
+
+class _Control(Enum):
+    OPEN = "<|open|>"
+    CLOSE = "<|close|>"
+    SEP = "<|sep|>"
+    END_OF_MSG = "<|end_of_msg|>"
+
+
+_Segment: TypeAlias = str | _Control
+_Segments: TypeAlias = tuple[_Segment, ...]
+
+
+def _open(tag: str, attrs: Iterable[tuple[str, str]] = ()) -> _Segments:
+    escaped: Final = (
+        (f" {key}", '="', value.replace("&", "&amp;").replace('"', "&quot;"), '"') for key, value in attrs
+    )
+    return (_Control.OPEN, tag, *chain.from_iterable(escaped), _Control.SEP)
+
+
+def _close(tag: str) -> _Segments:
+    return (_Control.CLOSE, tag, _Control.SEP)
+
+
+def _message(attrs: Iterable[tuple[str, str]], body: Iterable[_Segment]) -> _Segments:
+    return (*_open("message", attrs), *body, *_close("message"), _Control.END_OF_MSG)
+
+
+def _tool_declare(specs: Sequence[_ToolSpec]) -> _Segments:
+    tools: Final = json.dumps(
+        [{"type": "function", "function": spec.declared()} for spec in specs],
+        ensure_ascii=False,
+        separators=(",", ":"),
+        sort_keys=True,
+    )
+    return _message(
+        (("role", "system"), ("type", "tool-declare")),
+        (f"# Tools\nHere are the available tools, described in JSONSchema.\n\n```json\n{tools}\n```",),
+    )
+
+
+def _text(blocks: Iterable[_Converse]) -> str:
+    return "".join(block.text for block in blocks if isinstance(block, _Text))
+
+
+def _json_type(value: JsonValue) -> str:
+    match value:
+        case bool():
+            return "boolean"
+        case None:
+            return "null"
+        case int() | float():
+            return "number"
+        case str():
+            return "string"
+        case dict():
+            return "object"
+        case _:
+            return "array"
+
+
+def _argument(key: str, value: JsonValue) -> _Segments:
+    text: Final = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
+    return (*_open("argument", (("key", key), ("type", _json_type(value)))), text, *_close("argument"))
+
+
+def _call(index: int, call: _ToolUse) -> _Segments:
+    arguments: Final = chain.from_iterable(_argument(key, value) for key, value in call.arguments.items())
+    return (*_open("call", (("tool", call.name), ("index", str(index)))), *arguments, *_close("call"))
+
+
+def _tool_calls(calls: Sequence[_ToolUse]) -> _Segments:
+    if not calls:
+        return ()
+    rendered: Final = chain.from_iterable(_call(index, call) for index, call in enumerate(calls, start=1))
+    return (*_open("tools"), *rendered, *_close("tools"))
+
+
+def _assistant_message(message: _AssistantMessage) -> _Segments:
+    reasoning: Final = "".join(
+        block.reasoning_content.reasoning_text.text for block in message.content if isinstance(block, _ReasoningBlock)
+    )
+    return _message(
+        (("role", "assistant"),),
+        (
+            *_open("think"),
+            reasoning,
+            *_close("think"),
+            *_open("response"),
+            _text(message.content),
+            *_close("response"),
+            *_tool_calls(message.calls()),
+        ),
+    )
+
+
+def _tool_messages(results: Sequence[_ToolResult], previous: _AssistantMessage) -> _Segments:
+    names: Final = {call.tool_use_id: call.name for call in previous.calls()}
+    return tuple(
+        chain.from_iterable(
+            _message(
+                (("role", "tool"), ("tool", names[result.tool_use_id]), ("index", str(index))),
+                (_text(result.content),),
+            )
+            for index, result in enumerate(results, start=1)
+        )
+    )
+
+
+def _user_run(blocks: Sequence[_Text | _ToolResultBlock], previous: _Message | None) -> _Segments:
+    results: Final = tuple(block.tool_result for block in blocks if isinstance(block, _ToolResultBlock))
+    if results and isinstance(previous, _AssistantMessage):
+        return _tool_messages(results, previous)
+    return _message((("role", "user"),), (_text(blocks),))
+
+
+def _user_messages(message: _UserMessage, previous: _Message | None) -> _Segments:
+    blocks: Final = tuple(block for block in message.content if not isinstance(block, _CachePoint))
+    runs: Final = groupby(blocks, key=lambda block: isinstance(block, _ToolResultBlock))
+    return tuple(chain.from_iterable(_user_run(tuple(run), previous) for _, run in runs))
+
+
+def _turn(previous: _Message | None, message: _Message) -> _Segments:
+    if isinstance(message, _AssistantMessage):
+        return _assistant_message(message)
+    return _user_messages(message, previous)
+
+
+def _rendered(request: _ConverseRequest) -> _Segments:
+    specs: Final = (
+        ()
+        if request.tool_config is None
+        else tuple(block.tool_spec for block in request.tool_config.tools if isinstance(block, _ToolSpecBlock))
+    )
+    system: Final = _text(request.system)
+    turns: Final = chain.from_iterable(
+        _turn(previous, message) for previous, message in zip((None, *request.messages), request.messages, strict=False)
+    )
+    return (
+        *(_tool_declare(specs) if specs else ()),
+        *(_message((("role", "system"),), (system,)) if system else ()),
+        *turns,
+        *_open("message", (("role", "assistant"),)),
+        *_open("think"),
+    )
+
+
+def _text_tokens(encoding: tiktoken.Encoding, text: str) -> int:
+    spans: Final = (match.span() for match in _CHUNK.finditer(text))
+    return sum(len(encoding.encode_ordinary(text[start:end])) for start, end in spans)
+
+
+class _TextPart(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+    type: Literal["text"]
+
+
+class _TextOnlyMessage(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+    content: str | tuple[_TextPart, ...] | None = None
+
+
+def _text_only(message: AllMessageValues) -> bool:
+    try:
+        _TextOnlyMessage.model_validate(message)
+    except ValidationError:
+        return False
+    return True
+
+
+def _as_converse_input(tool: ChatCompletionToolParam) -> tuple[Mapping[str, object], ...]:
+    return chat_completion_tools_for(tool) if is_flat_responses_tool(tool) else (tool,)
+
+
+def _converse_request(
+    model: str, messages: Sequence[AllMessageValues], tools: Sequence[ChatCompletionToolParam] | None
+) -> _ConverseRequest | None:
+    if not all(_text_only(message) for message in messages):
+        return None
+    try:
+        chat_tools: Final = tuple(
+            chat_tool for tool in tools or () for chat_tool in _as_converse_input(tool)
+        )  # comprehension-ok: flattens each tool's translation
+        return _ConverseRequest.model_validate(
+            litellm.AmazonConverseConfig().transform_request(
+                model=model,
+                messages=copy.deepcopy(list(messages)),
+                optional_params={"tools": copy.deepcopy(list(chat_tools))} if chat_tools else {},
+                litellm_params={},
+                headers={},
+            )
+        )
+    except Exception:  # noqa: BLE001  # a body litellm cannot send to Bedrock, or cannot render, falls back to the estimate
+        return None
+
+
+def kimi_k3_tokens(
+    encoding: tiktoken.Encoding,
+    model: str,
+    messages: Sequence[AllMessageValues],
+    tools: Sequence[ChatCompletionToolParam] | None,
+) -> int | None:
+    request: Final = _converse_request(model, messages, tools)
+    if request is None:
+        return None
+    return sum(
+        1 if isinstance(segment, _Control) else _text_tokens(encoding, segment) for segment in _rendered(request)
+    )
+
+
+def kimi_k3_anthropic_tokens(encoding: tiktoken.Encoding, model: str, fields: Mapping[str, object]) -> int | None:
+    body: Final = {**fields, "model": model, "max_tokens": 1}
+    try:
+        translated, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
+            copy.deepcopy(body),  # pyright: ignore[reportArgumentType]  # raw /v1/messages fields
+            custom_llm_provider="bedrock",
+        )
+    except Exception:  # noqa: BLE001  # a request the adapter cannot translate falls back to the estimate
+        return None
+    match translated:
+        case {"tools": tools}:
+            return kimi_k3_tokens(encoding, model, translated["messages"], tools)
+        case _:
+            return kimi_k3_tokens(encoding, model, translated["messages"], None)
+
+
+def kimi_k3_template_tokens(message_count: int, has_tools: bool) -> int:
+    return 14 + 13 * max(message_count - 1, 0) + (40 if has_tools else 0)

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -17,7 +17,11 @@ from litellm.llms.anthropic.chat.transformation import (
     DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
     AnthropicConfig,
 )
-from litellm.llms.anthropic.common_utils import AnthropicModelInfo
+from litellm.llms.anthropic.common_utils import (
+    AnthropicModelInfo,
+    thinking_block_binding,
+    validated_object_mapping,
+)
 from litellm.llms.anthropic.pass_through.messages.transformation import (
     AnthropicMessagesConfig,
 )
@@ -49,7 +53,6 @@ from litellm.types.llms.anthropic import (
     ANTHROPIC_BETA_HEADER_VALUES,
     ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,
     ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
-    AnthropicThinkingParam,
 )
 from litellm.types.llms.bedrock import BedrockInvokeAnthropicMessagesRequest
 from litellm.types.llms.openai import AllMessageValues
@@ -82,6 +85,10 @@ class AmazonAnthropicClaudeMessagesConfig(
     def custom_llm_provider(self) -> str | None:
         return "bedrock"
 
+    @property
+    def _carries_thinking_block_binding(self) -> bool:
+        return type(self) is AmazonAnthropicClaudeMessagesConfig
+
     @property
     def beta_headers_provider(self) -> str:
         return self.custom_llm_provider or "bedrock"
@@ -349,18 +356,14 @@ class AmazonAnthropicClaudeMessagesConfig(
         if not isinstance(output_config, dict):
             output_config = {}
         output_config.setdefault("effort", self._effort_from_thinking_budget(budget_tokens))
-        thinking: Final = anthropic_messages_request.get("thinking")
+        thinking: Final = validated_object_mapping(anthropic_messages_request.get("thinking"))
-        display: Final = thinking.get("display") if isinstance(thinking, dict) else None
+        display: Final = thinking.get("display") if thinking is not None else None
         anthropic_messages_request["output_config"] = output_config
-        if display is None:
+        anthropic_messages_request["thinking"] = {
-            adaptive_thinking: Final[AnthropicThinkingParam] = {"type": "adaptive"}
+            **(thinking_block_binding(thinking) if self._carries_thinking_block_binding else {}),
-            anthropic_messages_request["thinking"] = adaptive_thinking
+            "type": "adaptive",
-        else:
+            **({} if display is None else {"display": display}),
-            adaptive_thinking_with_display: Final[AnthropicThinkingParam] = {
+        }
-                "type": "adaptive",
-                "display": display,
-            }
-            anthropic_messages_request["thinking"] = adaptive_thinking_with_display
         verbose_logger.debug(
             "Bedrock clear_thinking_20251015: injected adaptive thinking with effort=%s for model=%s",
             output_config.get("effort"),
@@ -701,8 +704,11 @@ class AmazonAnthropicClaudeMessagesConfig(
             and isinstance(translated_thinking, dict)
             and translated_thinking.get("type") == "adaptive"
         ):
-            thinking_with_display: Final[AnthropicThinkingParam] = {"type": "adaptive", "display": "updates"}
+            anthropic_messages_request["thinking"] = {
-            anthropic_messages_request["thinking"] = thinking_with_display
+                **(thinking_block_binding(translated_thinking) if self._carries_thinking_block_binding else {}),
+                "type": "adaptive",
+                "display": "updates",
+            }
         self._normalize_system_role_messages(anthropic_messages_request, model=model)
         #########################################################
         ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ###

litellm/llms/bedrock/responses/transformation.py

@@ -50,6 +50,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.bedrock.common_utils import (
+    BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX,
     BedrockError,
     bedrock_supports_openai_responses,
 )
@@ -76,6 +77,10 @@ IMAGE_BLOCK_KEYS: Final = ("content", "output")
 IMAGE_BLOCK_TYPES: Final = frozenset({"input_image", "computer_screenshot"})
 
 
+def _without_chat_completions_route(model: str) -> str:
+    return model.removeprefix(BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX)
+
+
 def resolve_bedrock_bearer_token(api_key: str | None) -> str | None:
     return api_key or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
 
@@ -168,9 +173,13 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig):
         The capability decision lives here rather than in the shared dispatch so that
         onboarding a model, or changing how the signal is read, stays inside the
         Bedrock adapter. ``None`` leaves the caller's existing behaviour untouched --
-        chat-only Bedrock models keep the Chat Completions bridge.
+        chat-only Bedrock models keep the Chat Completions bridge. The ``chat_completions/``
+        opt-in only moves Chat Completions calls off Converse, so a Responses call on such a
+        deployment still takes this surface instead of being bridged.
         """
-        if not bedrock_supports_openai_responses(model, litellm.model_cost):
+        if not model or not bedrock_supports_openai_responses(
+            _without_chat_completions_route(model), litellm.model_cost
+        ):
             return None
         return cls()
 
@@ -328,7 +337,7 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig):
                 rewritten_types,
             )
         return super().transform_responses_api_request(
-            model=model,
+            model=_without_chat_completions_route(model),
             input=normalized_input,
             response_api_optional_request_params=response_api_optional_request_params,
             litellm_params=litellm_params,

litellm/llms/cloudflare/decisions/transformation.py

@@ -0,0 +1,54 @@
+from collections.abc import Mapping
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+from litellm.secret_managers.main import (
+    get_secret_str,
+    normalize_nonempty_secret_str,
+)
+
+_RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+
+
+class CloudflareDecisionsConfig(BaseDecisionsConfig):
+    api_key_env = ("CLOUDFLARE_API_KEY",)
+    api_base_env = ("CLOUDFLARE_API_BASE",)
+
+    def get_default_api_base(self) -> str | None:
+        account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID"))
+        if account_id is None:
+            return None
+        return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run"
+
+    def missing_api_base_message(self, custom_llm_provider: str) -> str:
+        return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly"
+
+    def canonical_model(self, model: str) -> str:
+        if model.startswith("@cf/"):
+            return model
+        return f"@cf/cloudflare/{model}"
+
+    def request_model(self, model: str) -> str:
+        return model.rsplit("/", maxsplit=1)[-1]
+
+    def get_complete_url(self, api_base: str, model: str) -> str:
+        normalized_api_base: Final = api_base.rstrip("/")
+        if normalized_api_base.endswith("/ai/v1"):
+            return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}"
+        if normalized_api_base.endswith("/ai/run"):
+            return f"{normalized_api_base}/{model}"
+        return f"{normalized_api_base}/ai/run/{model}"
+
+    def unwrap_response(self, payload: object) -> object:
+        try:
+            response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload)
+        except ValidationError:
+            return payload
+        if "answers" in response_mapping:
+            return response_mapping
+        try:
+            return _RESPONSE_MAPPING_ADAPTER.validate_python(response_mapping.get("result"))
+        except ValidationError:
+            return response_mapping

litellm/llms/custom_httpx/llm_http_handler.py

@@ -3,7 +3,7 @@ import json
 import ssl
 from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Coroutine, Iterator, Mapping, Sequence
 from contextlib import asynccontextmanager
-from functools import lru_cache
+from functools import lru_cache, partial
 from types import MappingProxyType
 from typing import (
     TYPE_CHECKING,
@@ -68,6 +68,7 @@ from litellm.llms.base_llm.base_model_iterator import (
 from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
 from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
 from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig
 from litellm.llms.base_llm.files.transformation import (
@@ -120,6 +121,7 @@ from litellm.types.containers.main import (
     ContainerObject,
     DeleteContainerResult,
 )
+from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse
 from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig
 from litellm.types.integrations.custom_logger import (
     NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
@@ -1413,6 +1415,132 @@ class BaseLLMHTTPHandler:
             request_data=request_data,
         )
 
+    def _prepare_decisions_request(
+        self,
+        model: str,
+        logging_obj: LiteLLMLoggingObj | None,
+        provider_config: BaseDecisionsConfig,
+        body: Mapping[str, object],
+        api_base: str,
+        api_key: str | None,
+        headers: Mapping[str, str],
+    ) -> tuple[str, dict[str, str], dict[str, object]]:
+        outbound_headers: Final = provider_config.validate_environment(headers=headers, model=model, api_key=api_key)
+        url: Final = provider_config.get_complete_url(api_base=api_base, model=model)
+        data: Final = dict(body)
+        if logging_obj is not None:
+            logging_obj.pre_call(
+                input=data,
+                api_key=api_key,
+                model=model,
+                additional_args={"api_base": url, "complete_input_dict": data, "headers": outbound_headers},
+            )
+        return url, outbound_headers, data
+
+    def decisions(
+        self,
+        model: str,
+        custom_llm_provider: str,
+        logging_obj: LiteLLMLoggingObj | None,
+        provider_config: BaseDecisionsConfig,
+        request: DecisionsIRRequest,
+        body: Mapping[str, object],
+        api_base: str,
+        api_key: str | None,
+        headers: Mapping[str, str],
+        timeout: float | httpx.Timeout | None,
+        litellm_params: Mapping[str, object],
+        client: HTTPHandler | None = None,
+    ) -> DecisionsIRResponse:
+        url, outbound_headers, data = self._prepare_decisions_request(
+            model=model,
+            logging_obj=logging_obj,
+            provider_config=provider_config,
+            body=body,
+            api_base=api_base,
+            api_key=api_key,
+            headers=headers,
+        )
+        signed_headers, signed_body = provider_config.sign_request(
+            headers=outbound_headers,
+            url=url,
+            api_base=api_base,
+            body=data,
+            api_key=api_key,
+            litellm_params=litellm_params,
+        )
+        sync_httpx_client: Final = client if client is not None else _get_httpx_client()
+        try:
+            response: Final = sync_httpx_client.post(
+                url,
+                content=signed_body,
+                json=data if signed_body is None else None,
+                headers=dict(signed_headers),
+                timeout=timeout,
+                logging_obj=logging_obj,
+            )
+        except httpx.HTTPError as e:
+            raise self._handle_error(e=e, provider_config=provider_config)
+        return provider_config.transform_decisions_response(
+            model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request
+        )
+
+    async def adecisions(
+        self,
+        model: str,
+        custom_llm_provider: str,
+        logging_obj: LiteLLMLoggingObj | None,
+        provider_config: BaseDecisionsConfig,
+        request: DecisionsIRRequest,
+        body: Mapping[str, object],
+        api_base: str,
+        api_key: str | None,
+        headers: Mapping[str, str],
+        timeout: float | httpx.Timeout | None,
+        litellm_params: Mapping[str, object],
+        client: AsyncHTTPHandler | None = None,
+    ) -> DecisionsIRResponse:
+        url, outbound_headers, data = self._prepare_decisions_request(
+            model=model,
+            logging_obj=logging_obj,
+            provider_config=provider_config,
+            body=body,
+            api_base=api_base,
+            api_key=api_key,
+            headers=headers,
+        )
+        sign: Final = partial(
+            provider_config.sign_request,
+            headers=outbound_headers,
+            url=url,
+            api_base=api_base,
+            body=data,
+            api_key=api_key,
+            litellm_params=litellm_params,
+        )
+        signed_headers, signed_body = (
+            await run_aws_signing(sign) if provider_config.signs_with_aws(api_base) else sign()
+        )
+        async_httpx_client: Final = (
+            client
+            if client is not None
+            else get_async_httpx_client(llm_provider=litellm.LlmProviders(custom_llm_provider))
+        )
+        try:
+            response: Final = await async_httpx_client.post(
+                url,
+                content=signed_body,
+                json=data if signed_body is None else None,
+                headers=dict(signed_headers),
+                timeout=timeout,
+                logging_obj=logging_obj,
+            )
+        except httpx.HTTPError as e:
+            raise self._handle_error(e=e, provider_config=provider_config)
+        return provider_config.transform_decisions_response(
+            model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request
+        )
+
     def _prepare_audio_transcription_request(
         self,
         model: str,
@@ -1890,7 +2018,7 @@ class BaseLLMHTTPHandler:
         # str when the caller passes a pre-serialized (unsigned) body to avoid
         # re-dumping; bytes when a provider signed the request (e.g. Bedrock).
         signed_json_body: str | bytes | None,
-        request_body: dict,
+        request_body: dict[str, object],
         stream: bool,
         logging_obj: LiteLLMLoggingObj,
         provider_config: BaseAnthropicMessagesConfig,
@@ -1917,7 +2045,7 @@ class BaseLLMHTTPHandler:
             except httpx.HTTPStatusError as e:
                 hit_max_attempt = attempt_idx + 1 == max_attempts
                 should_retry = provider_config.should_retry_anthropic_messages_on_http_error(
-                    e=e, litellm_params=litellm_params_dict
+                    e=e, litellm_params=litellm_params_dict, request_body=request_body
                 )
                 if should_retry and not hit_max_attempt:
                     if logging_obj.baseline_cache_context is not None:
@@ -5926,6 +6054,7 @@ class BaseLLMHTTPHandler:
         e: Exception,
         provider_config: Union[
             BaseConfig,
+            BaseDecisionsConfig,
             BaseRerankConfig,
             BaseResponsesAPIConfig,
             BaseImageEditConfig,

litellm/llms/gemini/common_utils.py

@@ -6,6 +6,7 @@ from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
 import httpx
+from pydantic import JsonValue
 
 import litellm
 from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
@@ -13,7 +14,7 @@ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import TokenCountResponse
+from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
 
 GEMINI_IMAGE_ASPECT_RATIOS: Final[dict[str, float]] = {
     "1:1": 1 / 1,
@@ -491,21 +492,54 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
         request_model: str = "",
         tools: list[dict[str, object]] | None = None,
         system: object | None = None,
+        messages_format: TokenCountMessagesFormat = "openai",
+        output_format: JsonValue = None,
     ) -> TokenCountResponse | None:
         import copy
 
         from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter
+        from litellm.llms.vertex_ai.count_tokens.context import (
+            VertexGeminiCountContextError,
+            count_context_error_response,
+            gemini_count_contents,
+            inference_request_view,
+            provider_error_response,
+            unconvertible_messages_error,
+            vertex_gemini_count_context,
+        )
+
+        request_view: Final = inference_request_view(messages or [], system, messages_format)
+        count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
+        if isinstance(count_context, VertexGeminiCountContextError):
+            return count_context_error_response(count_context, request_model, model_to_use)
+        blocks_error: Final = unconvertible_messages_error(request_view.messages, messages_format)
+        if blocks_error is not None:
+            return count_context_error_response(blocks_error, request_model, model_to_use)
 
         deployment = deployment or {}
         count_tokens_params_request: Final = copy.deepcopy(deployment.get("litellm_params", {}))
+        resolved_contents: Final = (
+            contents
+            if contents is not None
+            else await gemini_count_contents(
+                request_view.messages, messages_format, model_to_use, "gemini", count_tokens_params_request
+            )
+        )
+        if isinstance(resolved_contents, VertexGeminiCountContextError):
+            return count_context_error_response(resolved_contents, request_model, model_to_use)
         count_tokens_params: Final = {
             "model": model_to_use,
-            "contents": contents,
+            "contents": resolved_contents,
+            "system_instruction": count_context.system_instruction,
+            "tools": count_context.tools,
         }
         count_tokens_params_request.update(count_tokens_params)
-        result: Final = await GoogleAIStudioTokenCounter().acount_tokens(
+        try:
-            **count_tokens_params_request,
+            result: Final = await GoogleAIStudioTokenCounter().acount_tokens(
-        )
+                **count_tokens_params_request,
+            )
+        except litellm.APIError as error:
+            return provider_error_response(error, request_model, model_to_use)
 
         if result is not None:
             return TokenCountResponse(

litellm/llms/gemini/count_tokens/handler.py

@@ -1,9 +1,10 @@
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
 
 import litellm
-from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
 from litellm.types.utils import LlmProviders
 
 if TYPE_CHECKING:
@@ -28,12 +29,12 @@ class GoogleAIStudioTokenCounter:
         """
         import copy
 
-        from google.genai.types import FunctionResponse
-
         # Handle None or empty contents
         if not contents:
             return contents
 
+        from google.genai.types import FunctionResponse
+
         cleaned_contents: Final = copy.deepcopy(contents)
 
         for content in cleaned_contents:
@@ -54,6 +55,28 @@ class GoogleAIStudioTokenCounter:
         base_url: Final = api_base or "https://generativelanguage.googleapis.com"
         return f"{base_url}/v1beta/models/{model}:countTokens"
 
+    def __init__(self, client: AsyncHTTPHandler | None = None) -> None:
+        super().__init__()
+        self._client: Final = client
+
+    def _build_count_tokens_request_body(
+        self,
+        model: str,
+        contents: object,
+        system_instruction: Mapping[str, object] | None,
+        tools: Sequence[Mapping[str, object]] | None,
+    ) -> Mapping[str, object]:
+        if system_instruction is None and not tools:
+            return {"contents": contents}
+
+        generate_content_request: Final = {
+            "model": model if model.startswith("models/") else f"models/{model}",
+            "contents": contents,
+            **({"systemInstruction": dict(system_instruction)} if system_instruction is not None else {}),
+            **({"tools": [dict(tool) for tool in tools]} if tools else {}),
+        }
+        return {"generateContentRequest": generate_content_request}
+
     async def validate_environment(
         self,
         api_base: str | None = None,
@@ -84,6 +107,8 @@ class GoogleAIStudioTokenCounter:
         api_key: str | None = None,
         api_base: str | None = None,
         timeout: float | httpx.Timeout | None = None,
+        system_instruction: Mapping[str, object] | None = None,
+        tools: Sequence[Mapping[str, object]] | None = None,
         **kwargs: object,
     ) -> dict[str, Any]:
         """
@@ -130,14 +155,19 @@ class GoogleAIStudioTokenCounter:
 
         # Prepare request body - clean up contents to remove unsupported fields
         cleaned_contents: Final = self._clean_contents_for_gemini_api(contents)
-        request_body: Final = {"contents": cleaned_contents}
+        request_body: Final = self._build_count_tokens_request_body(
+            model=model,
+            contents=cleaned_contents,
+            system_instruction=system_instruction,
+            tools=tools,
+        )
 
-        async_httpx_client: Final = get_async_httpx_client(
+        async_httpx_client: Final = (
-            llm_provider=LlmProviders.GEMINI,
+            self._client if self._client is not None else get_async_httpx_client(llm_provider=LlmProviders.GEMINI)
         )
 
         try:
-            response: Final = await async_httpx_client.post(url=url, headers=headers, json=request_body)
+            response: Final = await async_httpx_client.post(url=url, headers=headers, json=dict(request_body))
 
             # Check for HTTP errors
             response.raise_for_status()

litellm/llms/hosted_vllm/decisions/transformation.py

@@ -0,0 +1,26 @@
+from types import MappingProxyType
+from typing import Final
+
+from litellm.constants import PLACEHOLDER_API_KEY
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+
+
+class HostedVLLMDecisionsConfig(BaseDecisionsConfig):
+    api_key_env = ("HOSTED_VLLM_API_KEY",)
+    api_base_env = ("HOSTED_VLLM_API_BASE",)
+    api_key_required = False
+    health_check_questions = MappingProxyType(
+        {
+            "reachable": MappingProxyType(
+                {
+                    "type": "choice",
+                    "instructions": "Is the service reachable?",
+                    "criteria": {"yes": None, "no": None},
+                }
+            )
+        }
+    )
+
+    def resolve_api_key(self, api_key: str | None) -> str | None:
+        resolved: Final = super().resolve_api_key(api_key)
+        return None if resolved == PLACEHOLDER_API_KEY else resolved

litellm/llms/openai/decisions/transformation.py

@@ -0,0 +1,321 @@
+from collections.abc import Mapping, Sequence
+from typing import Final
+
+from pydantic import TypeAdapter
+from typing_extensions import assert_never
+
+import litellm
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.decisions import (
+    DecisionsIRAnswer,
+    DecisionsIRChoiceAnswer,
+    DecisionsIRChoiceOption,
+    DecisionsIRChoiceProbability,
+    DecisionsIRChoiceQuestion,
+    DecisionsIRMessages,
+    DecisionsIRPredicateAnswer,
+    DecisionsIRPredicateQuestion,
+    DecisionsIRQuestion,
+    DecisionsIRRefusal,
+    DecisionsIRRequest,
+    DecisionsIRResponse,
+    DecisionsIRScoreAnswer,
+    DecisionsIRScoreLevel,
+    DecisionsIRScoreProbability,
+    DecisionsIRScoreQuestion,
+    DecisionsIRState,
+    DecisionsIRUsage,
+    DecisionsJSON,
+    OpenAIChoiceAnswer,
+    OpenAIChoiceProbability,
+    OpenAIChoiceQuestion,
+    OpenAIDecisionAnswer,
+    OpenAIDecisionInput,
+    OpenAIDecisionInputTokensDetails,
+    OpenAIDecisionOutputTokensDetails,
+    OpenAIDecisionQuestion,
+    OpenAIDecisionRequestBody,
+    OpenAIDecisionResponse,
+    OpenAIDecisionUsage,
+    OpenAIPredicateAnswer,
+    OpenAIPredicateQuestion,
+    OpenAIRefusalAnswer,
+    OpenAIScoreAnswer,
+    OpenAIScoreProbability,
+    OpenAIScoreQuestion,
+    UnsupportedDecisionsRequest,
+    systemone_choice_key,
+)
+
+_OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse)
+_SINGLE_OPTION: Final = UnsupportedDecisionsRequest(
+    reason="OpenAI needs at least 2 choices or levels on every choice or score question"
+)
+
+
+def _ir_input(decision_input: OpenAIDecisionInput) -> DecisionsIRState | DecisionsIRMessages:
+    if isinstance(decision_input, str):
+        return DecisionsIRState(state=decision_input)
+    return DecisionsIRMessages(messages=tuple(decision_input))
+
+
+def _ir_question(question: OpenAIDecisionQuestion) -> DecisionsIRQuestion:
+    match question:
+        case OpenAIPredicateQuestion():
+            return DecisionsIRPredicateQuestion(name=question.name, instructions=question.instructions)
+        case OpenAIChoiceQuestion():
+            return DecisionsIRChoiceQuestion(
+                name=question.name,
+                instructions=question.instructions,
+                choices=tuple(
+                    DecisionsIRChoiceOption(value=option.value, description=option.description)
+                    for option in question.choices
+                ),
+            )
+        case OpenAIScoreQuestion():
+            return DecisionsIRScoreQuestion(
+                name=question.name,
+                instructions=question.instructions,
+                levels=tuple(
+                    DecisionsIRScoreLevel(label=level.label, description=level.description) for level in question.levels
+                ),
+            )
+        case _:
+            assert_never(question)
+
+
+def openai_request_to_ir(request: OpenAIDecisionRequestBody) -> DecisionsIRRequest:
+    return DecisionsIRRequest(
+        input=_ir_input(request.input),
+        questions=tuple(_ir_question(question) for question in request.questions),
+        safety_identifier=request.safety_identifier,
+    )
+
+
+def _text_field(key: str, value: DecisionsJSON | None) -> Mapping[str, str]:
+    return {} if value is None else {key: decisions_text(value)}
+
+
+def _instructions(value: DecisionsJSON | None, default: str) -> str:
+    return default if value is None else decisions_text(value)
+
+
+def _predicate_instructions(question: DecisionsIRPredicateQuestion) -> str:
+    instructions: Final = () if question.instructions is None else (decisions_text(question.instructions),)
+    criteria: Final = tuple(
+        f"Answer {answer} when: {decisions_text(rule)}"
+        for answer, rule in (question.criteria or {}).items()
+        if rule is not None
+    )
+    return "\n\n".join((*instructions, *criteria))
+
+
+def _openai_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
+    match question:
+        case DecisionsIRPredicateQuestion():
+            return {
+                "type": "predicate",
+                **_text_field("name", question.name),
+                "instructions": _predicate_instructions(question) or "Is this true of the input?",
+            }
+        case DecisionsIRChoiceQuestion():
+            return {
+                "type": "choice",
+                **_text_field("name", question.name),
+                "instructions": _instructions(question.instructions, "Which choice best fits the input?"),
+                "choices": [
+                    {"value": option.value, **_text_field("description", option.description)}
+                    for option in question.choices
+                ],
+            }
+        case DecisionsIRScoreQuestion():
+            return {
+                "type": "score",
+                **_text_field("name", question.name),
+                "instructions": _instructions(question.instructions, "Which level best fits the input?"),
+                "levels": [
+                    {"label": decisions_text(level.label), **_text_field("description", level.description)}
+                    for level in question.levels
+                ],
+            }
+        case _:
+            assert_never(question)
+
+
+def _openai_input(decision_input: DecisionsIRState | DecisionsIRMessages) -> str | Sequence[Mapping[str, object]]:
+    match decision_input:
+        case DecisionsIRState():
+            return decisions_text(decision_input.state)
+        case DecisionsIRMessages():
+            return [message.model_dump(mode="json", exclude_none=True) for message in decision_input.messages]
+        case _:
+            assert_never(decision_input)
+
+
+def _has_one_option(question: DecisionsIRQuestion) -> bool:
+    match question:
+        case DecisionsIRChoiceQuestion():
+            return len(question.choices) < 2
+        case DecisionsIRScoreQuestion():
+            return len(question.levels) < 2
+        case _:
+            return False
+
+
+def ir_to_openai_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object] | UnsupportedDecisionsRequest:
+    if any(_has_one_option(question) for question in request.questions):
+        return _SINGLE_OPTION
+    return {
+        "model": model,
+        "input": _openai_input(request.input),
+        "questions": [_openai_question(question) for question in request.questions],
+        **_text_field("safety_identifier", request.safety_identifier),
+    }
+
+
+def _level_label(question: DecisionsIRScoreQuestion, probability: OpenAIScoreProbability) -> DecisionsJSON:
+    if 0 <= probability.value < len(question.levels):
+        return question.levels[probability.value].label
+    return probability.label
+
+
+def _ir_answer(question: DecisionsIRQuestion, answer: OpenAIDecisionAnswer | None) -> DecisionsIRAnswer:
+    match question, answer:
+        case DecisionsIRPredicateQuestion(), OpenAIPredicateAnswer():
+            return DecisionsIRPredicateAnswer(probability=answer.probability)
+        case DecisionsIRChoiceQuestion(), OpenAIChoiceAnswer():
+            return DecisionsIRChoiceAnswer(
+                choice=answer.choice,
+                confidence=answer.confidence,
+                probabilities=tuple(
+                    DecisionsIRChoiceProbability(value=item.value, probability=item.probability)
+                    for item in answer.probabilities
+                ),
+            )
+        case DecisionsIRScoreQuestion(), OpenAIScoreAnswer():
+            return DecisionsIRScoreAnswer(
+                score=answer.score,
+                confidence=answer.confidence,
+                probabilities=tuple(
+                    DecisionsIRScoreProbability(
+                        value=item.value, label=_level_label(question, item), probability=item.probability
+                    )
+                    for item in answer.probabilities
+                ),
+            )
+        case _:
+            return DecisionsIRRefusal()
+
+
+def openai_response_to_ir(response: OpenAIDecisionResponse, request: DecisionsIRRequest) -> DecisionsIRResponse:
+    answers: Final = response.answers
+    return DecisionsIRResponse(
+        model=response.model,
+        answers=tuple(
+            _ir_answer(question, answers[index] if index < len(answers) else None)
+            for index, question in enumerate(request.questions)
+        ),
+        usage=DecisionsIRUsage(
+            input_tokens=response.usage.input_tokens,
+            output_tokens=response.usage.output_tokens,
+            cached_tokens=response.usage.input_tokens_details.cached_tokens,
+            cache_write_tokens=response.usage.input_tokens_details.cache_write_tokens,
+            reasoning_tokens=response.usage.output_tokens_details.reasoning_tokens,
+        ),
+    )
+
+
+def _openai_choice_answer(question: DecisionsIRChoiceQuestion, answer: DecisionsIRChoiceAnswer) -> OpenAIChoiceAnswer:
+    probabilities: Final = {systemone_choice_key(item.value): item.probability for item in answer.probabilities}
+    return OpenAIChoiceAnswer(
+        name=question.name,
+        choice=answer.choice,
+        probabilities=tuple(
+            OpenAIChoiceProbability(
+                value=option.value, probability=probabilities.get(systemone_choice_key(option.value), 0.0)
+            )
+            for option in question.choices
+        ),
+        confidence=answer.confidence,
+    )
+
+
+def _openai_score_answer(question: DecisionsIRScoreQuestion, answer: DecisionsIRScoreAnswer) -> OpenAIScoreAnswer:
+    probabilities: Final = {item.value: item.probability for item in answer.probabilities}
+    return OpenAIScoreAnswer(
+        name=question.name,
+        score=answer.score,
+        probabilities=tuple(
+            OpenAIScoreProbability(
+                value=index, label=decisions_text(level.label), probability=probabilities.get(index, 0.0)
+            )
+            for index, level in enumerate(question.levels)
+        ),
+        confidence=answer.confidence,
+    )
+
+
+def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) -> OpenAIDecisionAnswer:
+    match question, answer:
+        case DecisionsIRPredicateQuestion(), DecisionsIRPredicateAnswer():
+            return OpenAIPredicateAnswer(name=question.name, probability=answer.probability)
+        case DecisionsIRChoiceQuestion(), DecisionsIRChoiceAnswer():
+            return _openai_choice_answer(question, answer)
+        case DecisionsIRScoreQuestion(), DecisionsIRScoreAnswer():
+            return _openai_score_answer(question, answer)
+        case _:
+            return OpenAIRefusalAnswer(name=question.name)
+
+
+def ir_to_openai_response(
+    response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str
+) -> OpenAIDecisionResponse:
+    return OpenAIDecisionResponse(
+        model=response.model or requested_model,
+        answers=tuple(
+            _openai_answer(question, answer)
+            for question, answer in zip(request.questions, response.answers, strict=True)
+        ),
+        usage=OpenAIDecisionUsage(
+            input_tokens=response.usage.input_tokens,
+            input_tokens_details=OpenAIDecisionInputTokensDetails(
+                cached_tokens=response.usage.cached_tokens,
+                cache_write_tokens=response.usage.cache_write_tokens,
+            ),
+            output_tokens=response.usage.output_tokens,
+            output_tokens_details=OpenAIDecisionOutputTokensDetails(reasoning_tokens=response.usage.reasoning_tokens),
+            total_tokens=response.usage.input_tokens + response.usage.output_tokens,
+        ),
+    )
+
+
+class OpenAIDecisionsConfig(BaseDecisionsConfig):
+    path = "/v1/decisions"
+    supports_safety_identifier = True
+
+    def get_default_api_base(self) -> str | None:
+        return "https://api.openai.com"
+
+    def resolve_api_base(self, api_base: str | None) -> str | None:
+        return (
+            api_base
+            or litellm.api_base
+            or get_secret_str("OPENAI_BASE_URL")
+            or get_secret_str("OPENAI_API_BASE")
+            or self.get_default_api_base()
+        )
+
+    def resolve_api_key(self, api_key: str | None) -> str | None:
+        return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+
+    def transform_decisions_request(
+        self,
+        model: str,
+        request: DecisionsIRRequest,
+        custom_llm_provider: str,
+    ) -> Mapping[str, object] | UnsupportedDecisionsRequest:
+        return ir_to_openai_request(self.request_model(model), request)
+
+    def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
+        return openai_response_to_ir(_OPENAI_RESPONSE_ADAPTER.validate_python(payload), request)

litellm/llms/openai/responses/count_tokens/handler.py

@@ -5,6 +5,7 @@ Uses httpx for HTTP requests to OpenAI's /v1/responses/input_tokens endpoint.
 """
 
 import json
+from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
 import httpx
@@ -30,9 +31,10 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig):
         api_key: str,
         api_base: str | None = None,
         timeout: float | httpx.Timeout | None = None,
-        tools: list[dict[str, Any]] | None = None,
+        tools: Sequence[Mapping[str, object]] | None = None,
         instructions: str | None = None,
-    ) -> dict[str, Any]:
+        text: Mapping[str, object] | None = None,
+    ) -> dict[str, object]:
         """
         Handle a token counting request to OpenAI's Responses API.
 
@@ -52,6 +54,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig):
                 input=input,
                 tools=tools,
                 instructions=instructions,
+                text=text,
             )
 
             endpoint_url: Final = self.get_openai_count_tokens_endpoint(api_base)

litellm/llms/openai/responses/count_tokens/token_counter.py

@@ -2,24 +2,47 @@
 OpenAI Token Counter implementation using the Responses API /input_tokens endpoint.
 """
 
-import os
+from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
+import openai
+from pydantic import JsonValue
+from typing_extensions import assert_never
+
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.asyncify import asyncify
+from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error
+from litellm.llms.anthropic.pass_through.responses_adapters.transformation import (
+    LiteLLMAnthropicToResponsesAPIAdapter,
+)
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
-from litellm.llms.openai.common_utils import OpenAIError
+from litellm.llms.openai.common_utils import OpenAIError, get_openai_credentials
 from litellm.llms.openai.responses.count_tokens.handler import (
     OpenAICountTokensHandler,
 )
 from litellm.llms.openai.responses.count_tokens.transformation import (
     OpenAICountTokensConfig,
+    ResponsesCountRequestParts,
 )
-from litellm.types.utils import LlmProviders, TokenCountResponse
+from litellm.llms.openai.workload_identity import (
+    OpenAIWorkloadIdentityConfig,
+    get_workload_identity_bearer_token,
+    resolve_openai_workload_identity_config,
+)
+from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
 
 # Global handler instance - reuse across all token counting requests
 openai_count_tokens_handler: Final = OpenAICountTokensHandler()
 
 
+def _workload_identity_config(litellm_params: Mapping[str, object]) -> OpenAIWorkloadIdentityConfig | None:
+    api_base: Final = litellm_params.get("api_base")
+    return resolve_openai_workload_identity_config(
+        api_key=None,
+        api_base=api_base if isinstance(api_base, str) else None,
+    )
+
+
 class OpenAITokenCounter(BaseTokenCounter):
     """Token counter implementation for OpenAI provider using the Responses API."""
 
@@ -29,6 +52,37 @@ class OpenAITokenCounter(BaseTokenCounter):
     ) -> bool:
         return custom_llm_provider == LlmProviders.OPENAI.value
 
+    @staticmethod
+    def _system_instructions(system: object) -> str | None:
+        if system is None or isinstance(system, str):
+            return system
+        return LiteLLMAnthropicToResponsesAPIAdapter.system_blocks_to_instructions(system) or None
+
+    @staticmethod
+    def _responses_request_parts(
+        model_to_use: str,
+        messages: Sequence[Mapping[str, object]],
+        tools: Sequence[Mapping[str, object]] | None,
+        system: object,
+        messages_format: TokenCountMessagesFormat,
+        output_format: JsonValue,
+    ) -> ResponsesCountRequestParts:
+        match messages_format:
+            case "anthropic":
+                return OpenAICountTokensConfig.anthropic_messages_to_responses_input(
+                    model=model_to_use, messages=messages, system=system, tools=tools, output_format=output_format
+                )
+            case "openai":
+                input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(
+                    [dict(message) for message in messages]
+                )
+                fallback_instructions: Final = OpenAITokenCounter._system_instructions(system)
+                return OpenAICountTokensConfig.count_request_parts(
+                    input_items, instructions if instructions is not None else fallback_instructions, tools
+                )
+            case _:
+                assert_never(messages_format)
+
     async def count_tokens(
         self,
         model_to_use: str,
@@ -38,6 +92,8 @@ class OpenAITokenCounter(BaseTokenCounter):
         request_model: str = "",
         tools: list[dict[str, object]] | None = None,
         system: object = None,
+        messages_format: TokenCountMessagesFormat = "openai",
+        output_format: JsonValue = None,
     ) -> TokenCountResponse | None:
         """
         Count tokens using OpenAI's Responses API /input_tokens endpoint.
@@ -45,39 +101,76 @@ class OpenAITokenCounter(BaseTokenCounter):
         if not messages:
             return None
 
+        malformed_error: Final = (
+            OpenAICountTokensConfig.malformed_anthropic_messages_error(messages)
+            if messages_format == "anthropic"
+            else None
+        )
+        request_error: Final = malformed_error or unconvertible_blocks_error(messages, messages_format)
+        if request_error is not None:
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="openai_api",
+                error=True,
+                error_message=request_error,
+                status_code=400,
+                rejected_before_provider_call=True,
+            )
+
         deployment = deployment or {}
         litellm_params: Final = deployment.get("litellm_params", {})
 
-        # Get OpenAI API key from deployment config or environment
+        credentials: Final = get_openai_credentials(
-        api_key = litellm_params.get("api_key")
+            api_base=litellm_params.get("api_base"), api_key=litellm_params.get("api_key")
-        if not api_key:
+        )
-            api_key = os.getenv("OPENAI_API_KEY")
+        credential: Final = credentials.api_key or _workload_identity_config(litellm_params)
 
-        if not api_key:
+        if credential is None:
-            verbose_logger.warning("No OpenAI API key found for token counting")
+            return TokenCountResponse(
-            return None
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="openai_api",
+                error=True,
+                error_message=(
+                    f"No OpenAI API key is set for model {request_model or model_to_use}, so tokens can't be "
+                    "counted. Set api_key or workload identity on the deployment, or OPENAI_API_KEY in the proxy "
+                    "environment."
+                ),
+                status_code=401,
+            )
 
-        api_base: Final = litellm_params.get("api_base")
+        api_base: Final = credentials.api_base
 
-        # Convert chat messages to Responses API input format
+        input_items, instructions, counted_tools, text = self._responses_request_parts(
-        input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages)
+            model_to_use=model_to_use,
-
+            messages=messages,
-        # Use system param if instructions not extracted from messages
+            tools=tools,
-        if instructions is None and system is not None:
+            system=system,
-            instructions = system if isinstance(system, str) else str(system)
+            messages_format=messages_format,
+            output_format=output_format,
+        )
 
         # If no input items were produced (e.g., system-only messages), fall back to local counting
         if not input_items:
             return None
 
         try:
+            api_key: Final = (
+                credential
+                if isinstance(credential, str)
+                else await asyncify(get_workload_identity_bearer_token)(credential)
+            )
             result: Final = await openai_count_tokens_handler.handle_count_tokens_request(
                 model=model_to_use,
-                input=input_items if input_items is not None else [],
+                input=input_items,
                 api_key=api_key,
                 api_base=api_base,
-                tools=tools,
+                tools=counted_tools,
                 instructions=instructions,
+                text=text,
             )
 
             if result is not None:
@@ -99,6 +192,17 @@ class OpenAITokenCounter(BaseTokenCounter):
                 error_message=e.message,
                 status_code=e.status_code,
             )
+        except openai.AuthenticationError as e:
+            verbose_logger.warning("OpenAI workload identity token exchange failed: %s", e.message)
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="openai_api",
+                error=True,
+                error_message=e.message,
+                status_code=401,
+            )
         except Exception as e:
             verbose_logger.warning("Error calling OpenAI CountTokens API: %s", e)
             return TokenCountResponse(

litellm/llms/openai/responses/count_tokens/transformation.py

@@ -5,10 +5,18 @@ This module handles the transformation of requests to OpenAI's /v1/responses/inp
 """
 
 from collections.abc import Mapping, Sequence
-from typing import Any, Final, Literal
+from typing import Any, Final, Literal, TypeAlias
 
+from pydantic import JsonValue, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
+from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
+from litellm.llms.anthropic.pass_through.responses_adapters.transformation import (
+    LiteLLMAnthropicToResponsesAPIAdapter,
+)
+from litellm.responses.utils import ResponsesAPIRequestUtils
+from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicOutputSchema
+
 
 class ResponsesInputTextPart(TypedDict):
     type: ReadOnly[Literal["input_text"]]
@@ -31,6 +39,30 @@ ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart | Response
 
 ResponsesContentRole = Literal["user", "assistant"]
 
+ResponsesCountRequestParts: TypeAlias = tuple[
+    tuple[Mapping[str, object], ...],
+    str | None,
+    tuple[Mapping[str, object], ...] | None,
+    Mapping[str, object] | None,
+]
+_RESPONSES_COUNT_REQUEST_PARTS: Final = TypeAdapter(ResponsesCountRequestParts)
+_TRANSLATED_REQUEST: Final = TypeAdapter(Mapping[str, object])
+
+
+class _AnthropicCountMessage(TypedDict):
+    role: ReadOnly[str]
+
+
+_ANTHROPIC_COUNT_MESSAGES: Final = TypeAdapter(tuple[_AnthropicCountMessage, ...])
+_ANTHROPIC_OUTPUT_SCHEMA: Final = TypeAdapter(AnthropicOutputSchema)
+
+
+def _anthropic_output_schema(output_format: JsonValue) -> AnthropicOutputSchema | None:
+    try:
+        return _ANTHROPIC_OUTPUT_SCHEMA.validate_python(output_format)
+    except ValidationError:
+        return None
+
 
 def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImagePart | None:
     url: Final = image_url.get("url") if isinstance(image_url, Mapping) else image_url
@@ -118,8 +150,9 @@ class OpenAICountTokensConfig:
         self,
         model: str,
         input: str | Sequence[object],
-        tools: list[dict[str, Any]] | None = None,
+        tools: Sequence[Mapping[str, object]] | None = None,
         instructions: str | None = None,
+        text: Mapping[str, object] | None = None,
     ) -> dict[str, object]:
         """
         Transform request to OpenAI Responses API token counting format.
@@ -135,7 +168,10 @@ class OpenAICountTokensConfig:
             request["instructions"] = instructions
 
         if tools is not None:
-            request["tools"] = self._transform_tools_for_responses_api(tools)
+            request["tools"] = self._transform_tools_for_responses_api([dict(tool) for tool in tools])
+
+        if text is not None:
+            request["text"] = text
 
         return request
 
@@ -183,7 +219,7 @@ class OpenAICountTokensConfig:
     @staticmethod
     def messages_to_responses_input(
         messages: list[dict[str, Any]],
-    ) -> tuple:
+    ) -> tuple[list[dict[str, object]], str | None]:
         """
         Convert standard chat messages format to OpenAI Responses API input format.
 
@@ -246,3 +282,47 @@ class OpenAICountTokensConfig:
 
         instructions: Final = "\n".join(instructions_parts) if instructions_parts else None
         return input_items, instructions
+
+    @staticmethod
+    def malformed_anthropic_messages_error(messages: Sequence[Mapping[str, object]]) -> str | None:
+        try:
+            _ANTHROPIC_COUNT_MESSAGES.validate_python(messages)
+        except ValidationError as e:
+            return "; ".join(
+                f"messages.{'.'.join(str(part) for part in error['loc'])}: {error['msg']}" for error in e.errors()
+            )
+        return None
+
+    @staticmethod
+    def anthropic_messages_to_responses_input(
+        model: str,
+        messages: Sequence[Mapping[str, object]],
+        system: object,
+        tools: Sequence[Mapping[str, object]] | None,
+        output_format: JsonValue = None,
+    ) -> ResponsesCountRequestParts:
+        """Convert with the adapter /v1/messages inference uses, so the count sees the same input it bills."""
+        anthropic_request: Final = AnthropicMessagesRequest(
+            model=model,
+            messages=normalize_native_messages(messages),
+            system=system if isinstance(system, (str, list)) else None,
+            tools=[dict(tool) for tool in tools] if tools else None,
+            output_format=_anthropic_output_schema(output_format),
+        )
+        translated: Final = _TRANSLATED_REQUEST.validate_python(
+            LiteLLMAnthropicToResponsesAPIAdapter().translate_request(
+                anthropic_request, include_encrypted_reasoning=False
+            )
+        )
+        return OpenAICountTokensConfig.count_request_parts(
+            ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(translated.get("input")),  # pyright: ignore[reportPrivateUsage]  # the input restore inference runs
+            translated.get("instructions"),
+            translated.get("tools"),
+            translated.get("text"),
+        )
+
+    @staticmethod
+    def count_request_parts(
+        input_items: object, instructions: object, tools: object, text: object = None
+    ) -> ResponsesCountRequestParts:
+        return _RESPONSES_COUNT_REQUEST_PARTS.validate_python((input_items, instructions, tools, text))

litellm/llms/openrouter/chat/transformation.py

@@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, Final, cast
 
 import httpx
 
-import litellm
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
@@ -40,18 +39,10 @@ class CacheControlSupportedModels(str, Enum):
 class OpenrouterConfig(OpenAIGPTConfig):
     def get_supported_openai_params(self, model: str) -> list:
         """
-        Allow reasoning parameters for models flagged as reasoning-capable.
+        Include reasoning parameters regardless of model-map capabilities.
         """
         supported_params: Final = super().get_supported_openai_params(model=model)
-        try:
+        return list(dict.fromkeys([*supported_params, "reasoning_effort", "thinking"]))
-            if litellm.supports_reasoning(model=model, custom_llm_provider="openrouter") or litellm.supports_reasoning(
-                model=model
-            ):
-                supported_params.append("reasoning_effort")
-                supported_params.append("thinking")
-        except Exception:
-            pass
-        return list(dict.fromkeys(supported_params))
 
     def map_openai_params(
         self,

litellm/llms/openrouter/decisions/transformation.py

@@ -0,0 +1,19 @@
+from typing import Final
+
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+from litellm.types.decisions import DecisionsIRResponse
+
+
+class OpenRouterDecisionsConfig(BaseDecisionsConfig):
+    path = "/alpha/decisions"
+    api_key_env = ("OPENROUTER_API_KEY",)
+    api_base_env = ("OPENROUTER_API_BASE",)
+
+    def get_default_api_base(self) -> str | None:
+        return "https://openrouter.ai/api"
+
+    def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
+        cost: Final = response.usage.extra.get("cost")
+        if not isinstance(cost, (int, float)):
+            return None
+        return float(cost)

litellm/llms/perplexity/decisions/transformation.py

@@ -0,0 +1,10 @@
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+
+
+class PerplexityDecisionsConfig(BaseDecisionsConfig):
+    path = "/v1/decisions"
+    api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY")
+    api_base_env = ("PERPLEXITY_API_BASE",)
+
+    def get_default_api_base(self) -> str | None:
+        return "https://api.perplexity.ai"

litellm/llms/strands_decider/decisions/transformation.py

@@ -0,0 +1,161 @@
+import hashlib
+import json
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from functools import partial
+from types import MappingProxyType
+from typing import Final
+from urllib.parse import quote, unquote
+
+import httpx
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, sign_aws_json_post
+from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse
+from litellm.types.llms.bedrock import AwsAuthParams
+
+AGENTCORE_SESSION_HEADER: Final = "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"
+_RESERVED_HEADER_PREFIXES: Final = ("x-amzn-bedrock-agentcore-runtime-", "x-amz-")
+_RUNTIME_ARN: Final = re.compile(
+    r"\Aarn:aws(?:-[a-z]+)*:bedrock-agentcore:(?P<region>[a-z0-9-]+):[0-9]{12}:runtime/[A-Za-z0-9_-]+\Z"
+)
+_INVOCATIONS_PATH: Final = re.compile(r"\A/runtimes/(?P<arn>[^/]+)/invocations\Z")
+_RUNTIME_ERROR_STATUS: Final = MappingProxyType({"bad_request": 400, "invalid_request": 400, "loading": 503})
+
+
+@dataclass(frozen=True, slots=True)
+class AgentCoreRuntime:
+    arn: str
+    region: str
+
+    @property
+    def invocations_url(self) -> str:
+        host: Final = f"bedrock-agentcore.{self.region}.{get_aws_dns_suffix(self.region)}"
+        return f"https://{host}/runtimes/{quote(self.arn, safe='')}/invocations"
+
+    @property
+    def default_session_id(self) -> str:
+        return f"litellm-decider-{hashlib.sha256(self.arn.encode()).hexdigest()}"
+
+
+def agentcore_runtime(api_base: str) -> AgentCoreRuntime | None:
+    match: Final = _RUNTIME_ARN.match(api_base)
+    return None if match is None else AgentCoreRuntime(arn=api_base, region=match["region"])
+
+
+def invoked_runtime(url: httpx.URL) -> AgentCoreRuntime | None:
+    match: Final = _INVOCATIONS_PATH.match(url.raw_path.decode("ascii"))
+    runtime: Final = None if match is None else agentcore_runtime(unquote(match["arn"]))
+    return runtime if runtime is not None and runtime.invocations_url == str(url) else None
+
+
+class _RuntimeError(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    code: str
+    message: str = ""
+
+
+class _RuntimeReply(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    error: _RuntimeError | None = None
+    answers: object = None
+
+
+def _runtime_error(content: bytes) -> _RuntimeError | None:
+    try:
+        reply: Final = _RuntimeReply.model_validate_json(content)
+    except ValidationError:
+        return None
+    return reply.error if reply.answers is None else None
+
+
+class _SessionParams(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    agentcore_runtime_session_id: str | None = None
+
+
+def _is_reserved_header(name: str) -> bool:
+    lowered: Final = name.lower()
+    return lowered == "host" or lowered.startswith(_RESERVED_HEADER_PREFIXES)
+
+
+def _runtime_headers(
+    headers: Mapping[str, str], runtime: AgentCoreRuntime, litellm_params: Mapping[str, object]
+) -> Mapping[str, str]:
+    dropped: Final = sorted(name for name in headers if _is_reserved_header(name))
+    if dropped:
+        verbose_logger.warning("Strands Decider: not forwarding reserved AgentCore header(s) %s", dropped)
+    configured: Final = _SessionParams.model_validate(litellm_params).agentcore_runtime_session_id
+    return {
+        **{name: value for name, value in headers.items() if not _is_reserved_header(name)},
+        AGENTCORE_SESSION_HEADER: runtime.default_session_id if configured is None else configured,
+    }
+
+
+class StrandsDeciderDecisionsConfig(BaseDecisionsConfig):
+    api_key_env = ("STRANDS_DECIDER_API_KEY",)
+    api_base_env = ("STRANDS_DECIDER_API_BASE",)
+    api_key_required = False
+
+    def __init__(self, aws: BaseAWSLLM | None = None) -> None:
+        self._aws: Final = aws if aws is not None else BaseAWSLLM()
+
+    def get_complete_url(self, api_base: str, model: str) -> str:
+        runtime: Final = agentcore_runtime(api_base)
+        return super().get_complete_url(api_base, model) if runtime is None else runtime.invocations_url
+
+    def signs_with_aws(self, api_base: str) -> bool:
+        return agentcore_runtime(api_base) is not None
+
+    def sign_request(
+        self,
+        headers: Mapping[str, str],
+        url: str,
+        api_base: str,
+        body: Mapping[str, object],
+        api_key: str | None,
+        litellm_params: Mapping[str, object],
+    ) -> tuple[Mapping[str, str], bytes | None]:
+        runtime: Final = agentcore_runtime(api_base)
+        if runtime is None:
+            return headers, None
+        session_headers: Final = _runtime_headers(headers, runtime, litellm_params)
+        if api_key is not None:
+            return session_headers, None
+        payload: Final = json.dumps(body)
+        signed: Final = sign_aws_json_post(
+            get_credentials=partial(
+                self._aws.resolve_credentials, AwsAuthParams.model_validate(litellm_params), runtime.region
+            ),
+            service_name="bedrock-agentcore",
+            aws_region_name=runtime.region,
+            url=url,
+            body=payload,
+            headers=session_headers,
+        )
+        return dict(signed.headers.items()), payload.encode()
+
+    def transform_decisions_response(
+        self,
+        model: str,
+        custom_llm_provider: str,
+        raw_response: httpx.Response,
+        request: DecisionsIRRequest,
+    ) -> DecisionsIRResponse:
+        runtime_error: Final = (
+            None if invoked_runtime(raw_response.request.url) is None else _runtime_error(raw_response.content)
+        )
+        if runtime_error is not None:
+            raise BaseLLMException(
+                status_code=_RUNTIME_ERROR_STATUS.get(runtime_error.code, 500),
+                message=f"Strands Decider runtime error '{runtime_error.code}': {runtime_error.message}",
+            )
+        return super().transform_decisions_response(model, custom_llm_provider, raw_response, request)

litellm/llms/typesafe/decisions/transformation.py

@@ -0,0 +1,9 @@
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+
+
+class TypeSafeDecisionsConfig(BaseDecisionsConfig):
+    api_key_env = ("TYPESAFE_API_KEY",)
+    api_base_env = ("TYPESAFE_API_BASE",)
+
+    def get_default_api_base(self) -> str | None:
+        return "https://api.typesafe.ai"

litellm/llms/vertex_ai/common_utils.py

@@ -7,7 +7,7 @@ from types import MappingProxyType
 from typing import Any, Final, Literal, cast, get_type_hints
 
 import httpx
-from pydantic import TypeAdapter, ValidationError
+from pydantic import JsonValue, TypeAdapter, ValidationError
 from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
@@ -22,7 +22,7 @@ from litellm.types.llms.vertex_ai import (
     PartType,
     Schema,
 )
-from litellm.types.utils import TokenCountResponse
+from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
 from litellm.utils import supports_response_schema, supports_system_messages
 
 VERTEX_SELF_DEPLOYED_ENDPOINT_UNSUPPORTED_PARAMS: Final = frozenset(
@@ -1237,9 +1237,14 @@ class VertexAITokenCounter(BaseTokenCounter):
         request_model: str = "",
         tools: list[dict[str, object]] | None = None,
         system: object | None = None,
+        messages_format: TokenCountMessagesFormat = "openai",
+        output_format: JsonValue = None,
     ) -> TokenCountResponse | None:
         import copy
 
+        from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
+            VertexAIError as PartnerVertexAIError,
+        )
         from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
             VertexAIPartnerModels,
         )
@@ -1269,14 +1274,32 @@ class VertexAITokenCounter(BaseTokenCounter):
                 "vertex_ai_credentials"
             )
 
-            result = await partner_models_handler.count_tokens(
+            try:
-                model=model_to_use,
+                result = await partner_models_handler.count_tokens(
-                messages=messages or [],
+                    model=model_to_use,
-                litellm_params=partner_litellm_params,
+                    messages=messages or [],
-                vertex_project=vertex_project,
+                    litellm_params=partner_litellm_params,
-                vertex_location=vertex_location,
+                    vertex_project=vertex_project,
-                vertex_credentials=vertex_credentials,
+                    vertex_location=vertex_location,
-            )
+                    vertex_credentials=vertex_credentials,
+                    system=system,
+                    tools=tools,
+                )
+            except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
+                status_code: Final = e.response.status_code
+                error_message: Final = e.message if isinstance(e, PartnerVertexAIError) else e.response.text
+                verbose_logger.warning(
+                    "Vertex AI partner CountTokens API error: status=%s, message=%s", status_code, error_message
+                )
+                return TokenCountResponse(
+                    total_tokens=0,
+                    request_model=request_model,
+                    model_used=model_to_use,
+                    tokenizer_type="vertex_ai_partner_models",
+                    error=True,
+                    error_message=error_message,
+                    status_code=status_code,
+                )
 
             if result is not None:
                 return TokenCountResponse(
@@ -1287,30 +1310,55 @@ class VertexAITokenCounter(BaseTokenCounter):
                     original_response=result,
                 )
         else:
-            from litellm.llms.vertex_ai.count_tokens.handler import VertexAITokenCounter
+            from litellm.llms.vertex_ai.count_tokens.context import (
-            from litellm.llms.vertex_ai.gemini.transformation import (
+                VertexGeminiCountContextError,
-                _gemini_convert_messages_with_history,  # pyright: ignore[reportPrivateUsage]  # shared helper already used by gemini/chat, context_caching, and vertex_and_google_ai_studio_gemini
+                count_context_error_response,
+                gemini_count_contents,
+                inference_request_view,
+                provider_error_response,
+                unconvertible_messages_error,
+                vertex_gemini_count_context,
             )
+            from litellm.llms.vertex_ai.count_tokens.handler import VertexAITokenCounter
+
+            request_view: Final = inference_request_view(messages or [], system, messages_format)
+            count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
+            if isinstance(count_context, VertexGeminiCountContextError):
+                return count_context_error_response(count_context, request_model, model_to_use)
+            blocks_error: Final = unconvertible_messages_error(request_view.messages, messages_format)
+            if blocks_error is not None:
+                return count_context_error_response(blocks_error, request_model, model_to_use)
 
             resolved_contents: Final = (
-                contents if contents is not None else _gemini_convert_messages_with_history(messages=messages or [])
+                contents
+                if contents is not None
+                else await gemini_count_contents(
+                    request_view.messages, messages_format, model_to_use, "vertex_ai", count_tokens_params_request
+                )
             )
+            if isinstance(resolved_contents, VertexGeminiCountContextError):
+                return count_context_error_response(resolved_contents, request_model, model_to_use)
 
             count_tokens_params: Final = {
                 "model": model_to_use,
                 "contents": resolved_contents,
+                "system_instruction": count_context.system_instruction,
+                "tools": count_context.tools,
             }
             count_tokens_params_request.update(count_tokens_params)
-            result = await VertexAITokenCounter().acount_tokens(
+            try:
-                **count_tokens_params_request,
+                result = await VertexAITokenCounter().acount_tokens(
-            )
+                    **count_tokens_params_request,
+                )
+            except litellm.APIError as error:
+                return provider_error_response(error, request_model, model_to_use)
 
-            if result is not None and "totalTokens" in result:
+            if "totalTokens" in result:
                 return TokenCountResponse(
                     total_tokens=result["totalTokens"],
                     request_model=request_model,
                     model_used=model_to_use,
-                    tokenizer_type=result.get("tokenizer_used", ""),
+                    tokenizer_type=result.get("tokenizer_used", "gemini"),
                     original_response=result,
                 )
 

litellm/llms/vertex_ai/count_tokens/context.py

@@ -0,0 +1,242 @@
+import copy
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+from typing_extensions import assert_never
+
+from litellm.litellm_core_utils.prompt_templates.factory import chat_completion_tools_for
+from litellm.types.llms.anthropic import AnthropicMessagesTextParam
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.llms.vertex_ai import ContentType
+from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
+
+if TYPE_CHECKING:
+    from litellm.exceptions import APIError
+    from litellm.llms.anthropic.pass_through.context_management.result import PolyfillResult
+
+
+@dataclass(frozen=True, slots=True)
+class VertexGeminiCountContext:
+    system_instruction: Mapping[str, object] | None
+    tools: tuple[Mapping[str, object], ...] | None
+
+
+@dataclass(frozen=True, slots=True)
+class VertexGeminiCountContextError:
+    error_message: str
+
+
+_SYSTEM_BLOCKS: Final = TypeAdapter(tuple[AnthropicMessagesTextParam, ...])
+_TOOLS: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
+_VERTEX_TOOL_KEYS: Final = MappingProxyType(
+    {
+        "function_declarations": "functionDeclarations",
+        "url_context": "urlContext",
+        "code_execution": "codeExecution",
+    }
+)
+
+
+def _system_parts(
+    system: object | None,
+) -> tuple[Mapping[str, JsonValue], ...] | VertexGeminiCountContextError:
+    if system is None:
+        return ()
+    if isinstance(system, str):
+        return ({"text": system},) if system else ()
+    try:
+        blocks: Final = _SYSTEM_BLOCKS.validate_python(system)
+    except ValidationError:
+        return VertexGeminiCountContextError(error_message="Gemini system instructions only support text blocks.")
+    return tuple({"text": block["text"]} for block in blocks)
+
+
+_INVALID_TOOLS: Final = VertexGeminiCountContextError(
+    error_message="tools must be a list of objects, and each function tool needs a JSON object schema."
+)
+
+
+def _has_object_schema(tool: Mapping[str, JsonValue]) -> bool:
+    function: Final = tool.get("function")
+    schema: Final = (
+        function.get("parameters") if isinstance(function, dict) else tool.get("input_schema", tool.get("parameters"))
+    )
+    return schema is None or isinstance(schema, dict)
+
+
+def _is_function_tool(tool: Mapping[str, JsonValue]) -> bool:
+    return isinstance(tool.get("name"), str) or "input_schema" in tool or "function" in tool
+
+
+def _map_function_tools(
+    tools: object | None,
+) -> tuple[Mapping[str, object], ...] | None | VertexGeminiCountContextError:
+    try:
+        tool_values: Final = _TOOLS.validate_python(tools or ())
+    except ValidationError:
+        return _INVALID_TOOLS
+    function_tools: Final = tuple(tool for tool in tool_values if _is_function_tool(tool))
+    if not function_tools:
+        return None
+    if not all(_has_object_schema(tool) for tool in function_tools):
+        return _INVALID_TOOLS
+
+    from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
+
+    chat_tools: Final = tuple(
+        chat_tool for tool in function_tools for chat_tool in chat_completion_tools_for(tool)
+    )  # comprehension-ok: flattens each tool's translation
+    try:
+        mapped_tools: Final = VertexGeminiConfig()._map_function(  # pyright: ignore[reportPrivateUsage]  # the tool mapping inference uses
+            value=[dict(tool) for tool in chat_tools],
+            optional_params={},
+        )
+    except (AttributeError, TypeError, ValueError):
+        return _INVALID_TOOLS
+    return tuple(
+        MappingProxyType({_VERTEX_TOOL_KEYS.get(key, key): value for key, value in tool.items()})
+        for tool in mapped_tools
+    )
+
+
+def vertex_gemini_count_context(
+    system: object | None,
+    tools: object | None,
+) -> VertexGeminiCountContext | VertexGeminiCountContextError:
+    system_parts: Final = _system_parts(system)
+    if isinstance(system_parts, VertexGeminiCountContextError):
+        return system_parts
+    function_tools: Final = _map_function_tools(tools)
+    if isinstance(function_tools, VertexGeminiCountContextError):
+        return function_tools
+    return VertexGeminiCountContext(
+        system_instruction=MappingProxyType({"parts": system_parts}) if system_parts else None,
+        tools=function_tools,
+    )
+
+
+def unconvertible_messages_error(
+    messages: Sequence[Mapping[str, object]] | None, messages_format: TokenCountMessagesFormat
+) -> VertexGeminiCountContextError | None:
+    from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error
+
+    error_message: Final = unconvertible_blocks_error(messages, messages_format)
+    return None if error_message is None else VertexGeminiCountContextError(error_message=error_message)
+
+
+def count_context_error_response(
+    error: VertexGeminiCountContextError, request_model: str, model_used: str
+) -> TokenCountResponse:
+    return TokenCountResponse(
+        total_tokens=0,
+        request_model=request_model,
+        model_used=model_used,
+        tokenizer_type="gemini",
+        error=True,
+        error_message=error.error_message,
+        status_code=400,
+        rejected_before_provider_call=True,
+    )
+
+
+def provider_error_response(error: "APIError", request_model: str, model_used: str) -> TokenCountResponse:
+    return TokenCountResponse(
+        total_tokens=0,
+        request_model=request_model,
+        model_used=model_used,
+        tokenizer_type="gemini",
+        error=True,
+        error_message=error.message,
+        status_code=error.status_code,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class InferenceRequestView:
+    messages: Sequence[Mapping[str, object]]
+    system: object | None
+
+
+def _client_compaction(messages: Sequence[Mapping[str, object]], system: object | None) -> "PolyfillResult | None":
+    from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
+        apply_client_compaction_block_history,
+    )
+
+    chat_messages: Final = [dict(message) for message in messages]
+    if system is None or isinstance(system, str):
+        return apply_client_compaction_block_history(messages=chat_messages, system=system)
+    try:
+        blocks: Final = _SYSTEM_BLOCKS.validate_python(system)
+    except ValidationError:
+        return None
+    return apply_client_compaction_block_history(messages=chat_messages, system=[dict(block) for block in blocks])
+
+
+def inference_request_view(
+    messages: Sequence[Mapping[str, object]], system: object | None, messages_format: TokenCountMessagesFormat
+) -> InferenceRequestView:
+    """Apply the history rewrites /v1/messages runs before it converts a body for Gemini."""
+    from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
+
+    match messages_format:
+        case "openai":
+            return InferenceRequestView(messages=messages, system=system)
+        case "anthropic":
+            normalized: Final = normalize_native_messages(messages)
+            compacted: Final = _client_compaction(normalized, system)
+            if compacted is None:
+                return InferenceRequestView(messages=normalized, system=system)
+            return InferenceRequestView(messages=compacted.messages, system=compacted.system)
+        case _:
+            assert_never(messages_format)
+
+
+def _chat_messages(
+    messages: Sequence[Mapping[str, object]],
+    messages_format: TokenCountMessagesFormat,
+    model: str,
+    custom_llm_provider: Literal["vertex_ai", "gemini"],
+) -> Sequence[AllMessageValues]:
+    from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
+
+    match messages_format:
+        case "anthropic":
+            return LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
+                messages=list(messages),  # pyright: ignore[reportArgumentType]  # raw Anthropic body, read the way /v1/messages inference reads it
+                model=model,
+                custom_llm_provider=custom_llm_provider,
+            )
+        case "openai":
+            return copy.deepcopy(list(messages))  # pyright: ignore[reportReturnType]  # chat messages from the request, copied because the AI Studio conversion rewrites them in place
+        case _:
+            assert_never(messages_format)
+
+
+async def gemini_count_contents(
+    messages: Sequence[Mapping[str, object]],
+    messages_format: TokenCountMessagesFormat,
+    model: str,
+    custom_llm_provider: Literal["vertex_ai", "gemini"],
+    litellm_params: Mapping[str, object],
+) -> Sequence[ContentType] | VertexGeminiCountContextError:
+    from litellm.exceptions import BadRequestError
+    from litellm.litellm_core_utils.asyncify import asyncify
+    from litellm.litellm_core_utils.prompt_templates.image_handling import async_inline_remote_media
+    from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
+    from litellm.llms.vertex_ai.gemini.transformation import should_inline_remote_media
+    from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
+
+    config: Final = GoogleAIStudioGeminiConfig() if custom_llm_provider == "gemini" else VertexGeminiConfig()
+    try:
+        chat_messages: Final = await async_inline_remote_media(
+            list(_chat_messages(messages, messages_format, model, custom_llm_provider)),
+            should_inline=should_inline_remote_media(custom_llm_provider),
+        )
+        return await asyncify(config._transform_messages)(  # pyright: ignore[reportPrivateUsage]  # the message transform inference runs
+            messages=chat_messages, model=model, litellm_params=dict(litellm_params)
+        )
+    except BadRequestError as error:
+        return VertexGeminiCountContextError(error_message=error.message)

litellm/llms/vertex_ai/count_tokens/handler.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
 from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter
@@ -5,6 +6,19 @@ from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 
 
 class VertexAITokenCounter(GoogleAIStudioTokenCounter, VertexBase):
+    def _build_count_tokens_request_body(
+        self,
+        model: str,
+        contents: object,
+        system_instruction: Mapping[str, object] | None,
+        tools: Sequence[Mapping[str, object]] | None,
+    ) -> Mapping[str, object]:
+        return {
+            "contents": contents,
+            **({"systemInstruction": dict(system_instruction)} if system_instruction else {}),
+            **({"tools": [dict(tool) for tool in tools]} if tools else {}),
+        }
+
     async def validate_environment(
         self,
         api_base: str | None = None,
@@ -16,29 +30,29 @@ class VertexAITokenCounter(GoogleAIStudioTokenCounter, VertexBase):
         """
         Returns a Tuple of headers and url for the Vertex AI countTokens endpoint.
         """
-        litellm_params = litellm_params or {}
+        resolved_litellm_params: Final = litellm_params or {}
-        vertex_credentials: Final = self.get_vertex_ai_credentials(litellm_params=litellm_params)
+        vertex_credentials: Final = self.get_vertex_ai_credentials(litellm_params=resolved_litellm_params)
-        vertex_project = self.get_vertex_ai_project(litellm_params=litellm_params)
+        vertex_project: Final = self.get_vertex_ai_project(litellm_params=resolved_litellm_params)
-        vertex_location: Final = self.get_vertex_ai_location(litellm_params=litellm_params)
+        vertex_location: Final = self.get_vertex_ai_location(litellm_params=resolved_litellm_params)
-        _auth_header, vertex_project = await self._ensure_access_token_async(
+        _auth_header, resolved_vertex_project = await self._ensure_access_token_async(
             credentials=vertex_credentials,
             project_id=vertex_project,
             custom_llm_provider="vertex_ai",
         )
 
-        auth_header, api_base = self._get_token_and_url(
+        auth_header, url = self._get_token_and_url(
             model=model,
             gemini_api_key=None,
             auth_header=_auth_header,
-            vertex_project=vertex_project,
+            vertex_project=resolved_vertex_project,
             vertex_location=vertex_location,
             vertex_credentials=vertex_credentials,
             stream=False,
             custom_llm_provider="vertex_ai",
-            api_base=None,
+            api_base=api_base,
             mode="count_tokens",
         )
         headers = {
             "Authorization": f"Bearer {auth_header}",
         }
-        return headers, api_base
+        return headers, url

litellm/llms/vertex_ai/gemini/transformation.py

@@ -7,7 +7,7 @@ Why separate file? Make it easy to see how transformation works
 import json
 import os
 import re
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from typing import TYPE_CHECKING, Any, Final, Literal, cast
 from urllib.parse import quote
 
@@ -1327,6 +1327,10 @@ def _vertex_inlines(media: RemoteMedia) -> bool:
     )
 
 
+def should_inline_remote_media(custom_llm_provider: str) -> Callable[[RemoteMedia], bool]:
+    return _ai_studio_inlines if custom_llm_provider == "gemini" else _vertex_inlines
+
+
 async def async_transform_request_body(
     gemini_api_key: str | None,
     messages: list[AllMessageValues],
@@ -1369,7 +1373,7 @@ async def async_transform_request_body(
     )
 
     inlined_messages: Final = await async_inline_remote_media(
-        messages, should_inline=_ai_studio_inlines if custom_llm_provider == "gemini" else _vertex_inlines
+        messages, should_inline=should_inline_remote_media(custom_llm_provider)
     )
 
     if _openai_messages_may_need_sync_gcs_metadata_fetch(inlined_messages):

litellm/llms/vertex_ai/vertex_ai_partner_models/main.py

@@ -2,6 +2,7 @@
 ## API Handler for calling Vertex AI Partner Models
 from collections.abc import Callable
 from enum import Enum
+from types import MappingProxyType
 from typing import Final
 
 import httpx
@@ -263,6 +264,8 @@ class VertexAIPartnerModels(VertexBase):
         vertex_project=None,
         vertex_location=None,
         vertex_credentials=None,
+        system: object | None = None,
+        tools: list[dict[str, object]] | None = None,
     ):
         """
         Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.)
@@ -296,6 +299,9 @@ class VertexAIPartnerModels(VertexBase):
             request_data: Final = {
                 "model": model,
                 "messages": messages,
+                **MappingProxyType(
+                    {key: value for key, value in (("system", system), ("tools", tools)) if value is not None}
+                ),
             }
 
             # Prepare litellm_params with credentials

litellm/main.py

@@ -37,7 +37,7 @@ if TYPE_CHECKING:
 import dotenv
 import httpx
 import openai
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 from typing_extensions import assert_never, overload
 
 import litellm
@@ -116,7 +116,11 @@ from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig
 from litellm.llms.base_llm.base_model_iterator import (
     convert_model_response_to_streaming,
 )
-from litellm.llms.bedrock.common_utils import BedrockModelInfo
+from litellm.llms.bedrock.common_utils import (
+    BedrockModelInfo,
+    bedrock_route_for_request,
+    without_bedrock_route_prefix,
+)
 from litellm.llms.cohere.common_utils import CohereModelInfo
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
 from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
@@ -4167,6 +4171,10 @@ def _complete_sagemaker(ctx: _CompletionDispatchContext) -> _CompletionDispatchR
     )
 
 
+_ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
+_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
+
+
 def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
     acompletion: Final = ctx.acompletion
     api_base: Final = ctx.api_base
@@ -4205,7 +4213,12 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
         if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None:
             optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name
 
-    bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model)
+    additional_drop_params: Final = (
+        _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(ctx.kwargs["additional_drop_params"])
+        if ctx.kwargs.get("additional_drop_params") is not None
+        else None
+    )
+    bedrock_route: Final = bedrock_route_for_request(model, ctx.request_params, additional_drop_params)
     if bedrock_route == "claude_platform":
         provider_config = ProviderConfigManager.get_provider_chat_config(
             model=model,
@@ -4232,7 +4245,7 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
             provider_config=provider_config,
         )
     elif bedrock_route == "converse":
-        model = model.replace("converse/", "")
+        model = without_bedrock_route_prefix(model)
         response = bedrock_converse_chat_completion.completion(
             model=model,
             messages=messages,
@@ -5841,6 +5854,9 @@ def completion(
             optional_params=optional_params,
             organization=organization,
             provider_config=provider_config,
+            request_params=MappingProxyType(
+                _OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **non_default_params})
+            ),
             shared_session=shared_session,
             stream=stream,
             temperature=temperature,

litellm/model_prices_and_context_window_backup.json

@@ -19620,7 +19620,13 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": false,
+        "token_count_framing": 97
+    },
+    "databricks/system.ai.deepseek-v4-flash-0731": {
+        "litellm_provider": "databricks",
+        "mode": "chat",
+        "token_count_framing": 97
     },
     "databricks/databricks-deepseek-v4-pro-0813": {
         "cache_creation_input_token_cost": 1.31999e-06,
@@ -20611,7 +20617,13 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "token_count_framing": 100
+    },
+    "databricks/system.ai.kimi-k3": {
+        "litellm_provider": "databricks",
+        "mode": "chat",
+        "token_count_framing": 100
     },
     "databricks/databricks-llama-4-maverick": {
         "cache_creation_input_token_cost": 5.0001e-07,
@@ -34158,7 +34170,8 @@
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
-            "/v1/responses"
+            "/v1/responses",
+            "/v1/decisions"
         ],
         "supported_modalities": [
             "text",
@@ -41677,6 +41690,10 @@
         "output_cost_per_token": 0.0
     },
     "openai.gpt-oss-120b-1:0": {
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 128000,
@@ -41691,6 +41708,10 @@
         "supports_tool_choice": true
     },
     "openai.gpt-oss-20b-1:0": {
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
         "input_cost_per_token": 7e-08,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 128000,
@@ -47431,6 +47452,10 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 3.6e-07,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
         "supports_function_calling": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -47444,6 +47469,10 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
         "supports_function_calling": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -47453,6 +47482,11 @@
         "input_cost_per_token": 2.64e-06,
         "output_cost_per_token": 7.92e-06,
         "cache_read_input_token_cost": 6.6e-07,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 500000,
         "max_output_tokens": 500000,
@@ -58057,6 +58091,7 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html"
     },
     "us.openai.gpt-5.6-sol": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 4.4e-06,
         "input_cost_per_token_above_272k_tokens": 8.8e-06,
         "cache_creation_input_token_cost": 5.5e-06,
@@ -58087,10 +58122,12 @@
         "supports_vision": true,
         "supports_sampling_params": false,
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
     "global.openai.gpt-5.6-sol": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 4e-06,
         "input_cost_per_token_above_272k_tokens": 8e-06,
         "cache_creation_input_token_cost": 5e-06,
@@ -58121,10 +58158,12 @@
         "supports_vision": true,
         "supports_sampling_params": false,
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
     "us.openai.gpt-5.6-terra": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 2.2e-06,
         "input_cost_per_token_above_272k_tokens": 4.4e-06,
         "cache_creation_input_token_cost": 2.75e-06,
@@ -58155,10 +58194,12 @@
         "supports_vision": true,
         "supports_sampling_params": false,
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
     "global.openai.gpt-5.6-terra": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 2e-06,
         "input_cost_per_token_above_272k_tokens": 4e-06,
         "cache_creation_input_token_cost": 2.5e-06,
@@ -58189,10 +58230,12 @@
         "supports_vision": true,
         "supports_sampling_params": false,
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
     "us.openai.gpt-5.6-luna": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 2.2e-07,
         "input_cost_per_token_above_272k_tokens": 4.4e-07,
         "cache_creation_input_token_cost": 2.75e-07,
@@ -58223,6 +58266,7 @@
         "supports_vision": true,
         "supports_sampling_params": false,
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
@@ -58351,6 +58395,7 @@
         ]
     },
     "global.openai.gpt-5.6-luna": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 2e-07,
         "input_cost_per_token_above_272k_tokens": 4e-07,
         "cache_creation_input_token_cost": 2.5e-07,
@@ -58381,6 +58426,7 @@
         "supports_vision": true,
         "supports_sampling_params": false,
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
@@ -58499,6 +58545,7 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html"
     },
     "us.openai.gpt-6-astra": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 1.1e-05,
         "input_cost_per_token_above_272k_tokens": 2.2e-05,
         "cache_creation_input_token_cost": 1.375e-05,
@@ -58528,12 +58575,15 @@
         "supports_reasoning": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "source": "https://aws.amazon.com/bedrock/pricing/",
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
     "us.openai.gpt-6-sol": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 2.2e-06,
         "input_cost_per_token_above_272k_tokens": 4.4e-06,
         "cache_creation_input_token_cost": 2.75e-06,
@@ -58563,12 +58613,15 @@
         "supports_reasoning": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "source": "https://aws.amazon.com/bedrock/pricing/",
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
     "us.openai.gpt-6-luna": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 1.1e-07,
         "input_cost_per_token_above_272k_tokens": 2.2e-07,
         "cache_creation_input_token_cost": 1.375e-07,
@@ -58598,12 +58651,15 @@
         "supports_reasoning": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "source": "https://aws.amazon.com/bedrock/pricing/",
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
     "global.openai.gpt-6-astra": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 1e-05,
         "input_cost_per_token_above_272k_tokens": 2e-05,
         "cache_creation_input_token_cost": 1.25e-05,
@@ -58633,8 +58689,10 @@
         "supports_reasoning": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "source": "https://aws.amazon.com/bedrock/pricing/",
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
@@ -58668,9 +58726,11 @@
         "supports_reasoning": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "source": "https://aws.amazon.com/bedrock/pricing/"
     },
     "global.openai.gpt-6-sol": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 2e-06,
         "input_cost_per_token_above_272k_tokens": 4e-06,
         "cache_creation_input_token_cost": 2.5e-06,
@@ -58700,8 +58760,10 @@
         "supports_reasoning": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "source": "https://aws.amazon.com/bedrock/pricing/",
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
@@ -58735,9 +58797,11 @@
         "supports_reasoning": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "source": "https://aws.amazon.com/bedrock/pricing/"
     },
     "global.openai.gpt-6-luna": {
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "input_cost_per_token": 1e-07,
         "input_cost_per_token_above_272k_tokens": 2e-07,
         "cache_creation_input_token_cost": 1.25e-07,
@@ -58767,8 +58831,10 @@
         "supports_reasoning": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "source": "https://aws.amazon.com/bedrock/pricing/",
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ]
     },
@@ -59065,6 +59131,11 @@
         "input_cost_per_token": 2.2e-06,
         "output_cost_per_token": 6.6e-06,
         "cache_read_input_token_cost": 5.5e-07,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 500000,
         "max_output_tokens": 500000,
@@ -59081,6 +59152,11 @@
         "input_cost_per_token": 2e-06,
         "output_cost_per_token": 6e-06,
         "cache_read_input_token_cost": 5e-07,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 500000,
         "max_output_tokens": 500000,
@@ -65067,6 +65143,10 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 3.6e-07,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
         "supports_function_calling": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -65080,6 +65160,10 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
         "supports_function_calling": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -65321,6 +65405,10 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 3.6e-07,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
         "supports_function_calling": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -65334,6 +65422,10 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 7.2e-07,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
         "supports_function_calling": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -76700,7 +76792,9 @@
         "supports_response_schema": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "token_count_framing": 64,
+        "token_count_tokenizer": "kimi_k3"
     },
     "global.moonshotai.kimi-k3": {
         "cache_creation_input_token_cost": 3.75e-06,
@@ -76720,7 +76814,9 @@
         "supports_response_schema": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "token_count_framing": 64,
+        "token_count_tokenizer": "kimi_k3"
     },
     "us.moonshotai.kimi-k3": {
         "cache_creation_input_token_cost": 4.125e-06,
@@ -76740,7 +76836,9 @@
         "supports_response_schema": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "token_count_framing": 64,
+        "token_count_tokenizer": "kimi_k3"
     },
     "openrouter/xiaomi/mimo-v2.6-flash": {
         "cache_read_input_token_cost": 2.8e-09,
@@ -79457,6 +79555,7 @@
         "output_cost_per_token_above_272k_tokens": 1.5e-05,
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html",
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ],
         "supported_modalities": [
@@ -79466,6 +79565,7 @@
         "supported_output_modalities": [
             "text"
         ],
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "supports_function_calling": true,
         "supports_max_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
@@ -79474,6 +79574,7 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "supports_xhigh_reasoning_effort": true
     },
     "openai.gpt-6.1-sol": {
@@ -79506,6 +79607,7 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "supports_xhigh_reasoning_effort": true
     },
     "bedrock_mantle/openai.gpt-6.1-sol": {
@@ -79562,6 +79664,7 @@
         "output_cost_per_token_above_272k_tokens": 1.65e-05,
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html",
         "supported_endpoints": [
+            "/v1/chat/completions",
             "/v1/responses"
         ],
         "supported_modalities": [
@@ -79571,6 +79674,7 @@
         "supported_output_modalities": [
             "text"
         ],
+        "supports_bedrock_runtime_chat_completions_response_format": true,
         "supports_function_calling": true,
         "supports_max_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
@@ -79579,6 +79683,7 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": true,
+        "supports_sampling_params": false,
         "supports_xhigh_reasoning_effort": true
     },
     "vertex_ai/gemini-3.8-flash-tts": {
@@ -79628,5 +79733,62 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true
+    },
+    "cloudflare/clef": {
+        "input_cost_per_token": 2.4e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 65536,
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
+    },
+    "cloudflare/clef-flash": {
+        "input_cost_per_token": 9e-08,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 65536,
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
+    },
+    "cloudflare/@cf/cloudflare/clef": {
+        "input_cost_per_token": 2.4e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 65536,
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
+    },
+    "cloudflare/@cf/cloudflare/clef-flash": {
+        "input_cost_per_token": 9e-08,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 65536,
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/"
+    },
+    "perplexity/pplx-decider-v1-27b": {
+        "input_cost_per_token": 4e-08,
+        "litellm_provider": "perplexity",
+        "max_input_tokens": 262144,
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.perplexity.ai/api-reference/decisions-post"
+    },
+    "strands_decider/strands-decider-2B-hobson-v19": {
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "strands_decider",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v19",
+        "metadata": {
+            "notes": "Self-hosted decision model; infrastructure costs are paid separately"
+        }
+    },
+    "azure_ai/Microsoft-Decision-1": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "azure_ai",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'"
     }
 }

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -142,6 +142,12 @@ from litellm.proxy._experimental.mcp_server.result_conversion import (
 from litellm.proxy._experimental.mcp_server.sampling_handler import (
     MCP_SAMPLING_AVAILABLE,
 )
+from litellm.proxy._experimental.mcp_server.stdio_gate import (
+    MCP_STDIO_DISABLED_MESSAGE,
+    is_mcp_stdio_blocked,
+    is_mcp_stdio_enabled,
+    warn_if_mcp_stdio_blocked,
+)
 from litellm.proxy._experimental.mcp_server.tool_catalog_guard import (
     CatalogAlert,
     apply_description_overrides,
@@ -2464,6 +2470,7 @@ class MCPServerManager:
                 alias=alias,
                 server_name=server_name,
             )
+            warn_if_mcp_stdio_blocked(server_name, server_config.get("transport"))
 
             auth_type = server_config.get("auth_type", None)
             manual_issuer = _blank_to_none(server_config.get("issuer"))
@@ -3282,6 +3289,7 @@ class MCPServerManager:
                 # `credentials` field is the only one still encrypted here).
                 # Re-decrypting plaintext would zero the values, so build with
                 # env_vars_are_encrypted=False.
+                self._warn_if_newly_blocked_stdio(mcp_server, None)
                 new_server: Final = await self.build_mcp_server_from_table(mcp_server, env_vars_are_encrypted=False)
                 self._assign_unique_short_prefix(new_server)
                 self._invalidate_server_definition_caches(mcp_server.server_id)
@@ -4280,6 +4288,8 @@ class MCPServerManager:
 
         # Handle stdio transport
         if transport == MCPTransport.stdio:
+            if not is_mcp_stdio_enabled():
+                raise HTTPException(status_code=403, detail=MCP_STDIO_DISABLED_MESSAGE)
             resolved_env: Final = (
                 stdio_env
                 if stdio_env is not None
@@ -4439,6 +4449,9 @@ class MCPServerManager:
             global_mcp_tool_registry,
         )
 
+        if self._skip_blocked_stdio_listing(server, "tool"):
+            return []
+
         verbose_logger.debug("Connecting to url: %s", server.url)
         verbose_logger.info("_get_tools_from_server for %s...", server.name)
 
@@ -4638,6 +4651,19 @@ class MCPServerManager:
         )
         return server.server_id, hashlib.sha256(material.encode()).hexdigest()
 
+    @staticmethod
+    def _warn_if_newly_blocked_stdio(row: LiteLLM_MCPServerTable, previous: MCPServer | None) -> None:
+        if previous is None or previous.transport != row.transport:
+            warn_if_mcp_stdio_blocked(row.alias or row.server_name, row.transport)
+
+    def _skip_blocked_stdio_listing(self, server: MCPServer, listing: str) -> bool:
+        if not is_mcp_stdio_blocked(server.transport):
+            return False
+        verbose_logger.debug(
+            "Skipping %s listing for MCP server %s: %s", listing, server.name, MCP_STDIO_DISABLED_MESSAGE
+        )
+        return True
+
     async def get_prompts_from_server(
         self,
         server: MCPServer,
@@ -4648,6 +4674,8 @@ class MCPServerManager:
         raw_headers: dict[str, str] | None = None,
         client_ip: str | None = None,
     ) -> list[Prompt]:
+        if self._skip_blocked_stdio_listing(server, "prompt"):
+            return []
         try:
             headers: Final = (
                 dict(
@@ -4694,6 +4722,8 @@ class MCPServerManager:
         raw_headers: dict[str, str] | None = None,
         client_ip: str | None = None,
     ) -> list[Resource]:
+        if self._skip_blocked_stdio_listing(server, "resource"):
+            return []
         try:
             headers: Final = (
                 dict(
@@ -4740,6 +4770,8 @@ class MCPServerManager:
         raw_headers: dict[str, str] | None = None,
         client_ip: str | None = None,
     ) -> list[ResourceTemplate]:
+        if self._skip_blocked_stdio_listing(server, "resource template"):
+            return []
         try:
             headers: Final = (
                 dict(
@@ -6324,6 +6356,8 @@ class MCPServerManager:
                 mcp_server = fallback
         if mcp_server is None:
             raise ValueError(f"Tool {name} not found")
+        if is_mcp_stdio_blocked(mcp_server.transport):
+            raise HTTPException(status_code=403, detail=MCP_STDIO_DISABLED_MESSAGE)
 
         if resolved_by_server_name_only and not self.server_exposes_tool(mcp_server, name):
             raise ValueError(f"Tool {name} not found")
@@ -6708,7 +6742,10 @@ class MCPServerManager:
         if matched is not None:
             matched_prefix, original_tool_name = matched
             matched_server: Final = prefix_to_server.get(matched_prefix)
-            if matched_server is not None and self.server_exposes_tool(matched_server, original_tool_name):
+            if matched_server is not None and (
+                self.server_exposes_tool(matched_server, original_tool_name)
+                or is_mcp_stdio_blocked(matched_server.transport)
+            ):
                 return matched_server
 
         return None
@@ -6770,6 +6807,7 @@ class MCPServerManager:
                     alias=getattr(server, "alias", None),
                     server_name=getattr(server, "server_name", None),
                 )
+                self._warn_if_newly_blocked_stdio(server, existing_server)
                 verbose_logger.debug("Building server from DB: %s (%s)", server.server_id, server.server_name)
                 # raw_rows come straight from the DB, so their global env var
                 # values (like credentials) are still encrypted here, unlike the

litellm/proxy/_experimental/mcp_server/stdio_gate.py

@@ -0,0 +1,28 @@
+import os
+from typing import Final
+
+from litellm._logging import verbose_logger
+from litellm.types.mcp import MCPTransport
+
+MCP_STDIO_ENABLED_ENV_VAR: Final = "LITELLM_ENABLE_MCP_STDIO"
+MCP_STDIO_DISABLED_MESSAGE: Final = (
+    f"stdio MCP servers are disabled on this proxy. "
+    f"Set {MCP_STDIO_ENABLED_ENV_VAR}=true on the proxy and restart to enable them"
+)
+
+
+def is_mcp_stdio_enabled() -> bool:
+    return os.getenv(MCP_STDIO_ENABLED_ENV_VAR, "").strip().lower() == "true"
+
+
+def is_mcp_stdio_flag_key(env_var_name: str) -> bool:
+    return env_var_name.upper() == MCP_STDIO_ENABLED_ENV_VAR
+
+
+def is_mcp_stdio_blocked(transport: str | None) -> bool:
+    return transport == MCPTransport.stdio and not is_mcp_stdio_enabled()
+
+
+def warn_if_mcp_stdio_blocked(server_name: str | None, transport: str | None) -> None:
+    if is_mcp_stdio_blocked(transport):
+        verbose_logger.warning("MCP server '%s' will not start: %s", server_name, MCP_STDIO_DISABLED_MESSAGE)

litellm/proxy/_experimental/out/__next._tree.txt

@@ -1,4 +1,4 @@
-:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
+:HL["/litellm-asset-prefix/_next/static/chunks/0ckv5j_fni3xm.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/3146e697tym4_.css","style"]
+:HL["/litellm-asset-prefix/_next/static/chunks/2cd3v4aaz6o0i.css","style"]
 :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
-0:{"tree":{"name":"","param":null,"prefetchHints":4176,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":4192,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":4256,"slots":null}}}}},"staleTime":300,"buildId":"kXnLzJ6ylsRPmgSkCkCKM"}
+0:{"tree":{"name":"","param":null,"prefetchHints":4176,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":4192,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":4256,"slots":null}}}}},"staleTime":300,"buildId":"vrifX6k3UPQnwaiDi5xVQ"}